2024-11-25 12:45:49,238 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@6f15d60e 2024-11-25 12:45:49,290 main DEBUG Took 0.048471 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 12:45:49,290 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 12:45:49,292 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 12:45:49,293 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 12:45:49,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,313 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 12:45:49,348 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,352 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,354 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,355 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,357 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,358 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,360 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,361 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,362 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,363 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,365 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:45:49,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,366 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 12:45:49,368 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:45:49,370 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 12:45:49,373 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 12:45:49,374 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 12:45:49,376 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 12:45:49,376 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 12:45:49,390 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 12:45:49,397 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 12:45:49,400 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 12:45:49,400 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 12:45:49,401 main DEBUG createAppenders(={Console}) 2024-11-25 12:45:49,402 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@6f15d60e initialized 2024-11-25 12:45:49,403 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@6f15d60e 2024-11-25 12:45:49,403 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@6f15d60e OK. 2024-11-25 12:45:49,404 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 12:45:49,405 main DEBUG OutputStream closed 2024-11-25 12:45:49,405 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 12:45:49,406 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 12:45:49,406 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@26e356f0 OK 2024-11-25 12:45:49,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 12:45:49,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 12:45:49,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 12:45:49,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 12:45:49,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 12:45:49,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 12:45:49,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 12:45:49,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 12:45:49,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 12:45:49,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 12:45:49,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 12:45:49,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 12:45:49,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 12:45:49,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 12:45:49,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 12:45:49,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 12:45:49,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 12:45:49,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 12:45:49,530 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 12:45:49,531 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@7e6ef134) with optional ClassLoader: null 2024-11-25 12:45:49,531 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 12:45:49,533 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@7e6ef134] started OK. 2024-11-25T12:45:49,874 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243 2024-11-25 12:45:49,877 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 12:45:49,878 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T12:45:49,890 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-25T12:45:49,900 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-25T12:45:49,937 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T12:45:50,002 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T12:45:50,003 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T12:45:50,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T12:45:50,045 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a, deleteOnExit=true 2024-11-25T12:45:50,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T12:45:50,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/test.cache.data in system properties and HBase conf 2024-11-25T12:45:50,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T12:45:50,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir in system properties and HBase conf 2024-11-25T12:45:50,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T12:45:50,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T12:45:50,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T12:45:50,168 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T12:45:50,293 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T12:45:50,297 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T12:45:50,297 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T12:45:50,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T12:45:50,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T12:45:50,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T12:45:50,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T12:45:50,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T12:45:50,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T12:45:50,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T12:45:50,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/nfs.dump.dir in system properties and HBase conf 2024-11-25T12:45:50,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/java.io.tmpdir in system properties and HBase conf 2024-11-25T12:45:50,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T12:45:50,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T12:45:50,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T12:45:51,391 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T12:45:51,459 INFO [Time-limited test {}] log.Log(170): Logging initialized @3438ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T12:45:51,521 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:45:51,582 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:45:51,609 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:45:51,609 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:45:51,610 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T12:45:51,627 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:45:51,631 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dc5014{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:45:51,632 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a6f93cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:45:51,854 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c8b857b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/java.io.tmpdir/jetty-localhost-44043-hadoop-hdfs-3_4_1-tests_jar-_-any-17550271405127794171/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T12:45:51,860 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@58a7ff93{HTTP/1.1, (http/1.1)}{localhost:44043} 2024-11-25T12:45:51,860 INFO [Time-limited test {}] server.Server(415): Started @3839ms 2024-11-25T12:45:52,418 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:45:52,426 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:45:52,427 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:45:52,427 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:45:52,427 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T12:45:52,428 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:45:52,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:45:52,527 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@17d1a8d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/java.io.tmpdir/jetty-localhost-34879-hadoop-hdfs-3_4_1-tests_jar-_-any-11466394765148469199/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:45:52,528 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:34879} 2024-11-25T12:45:52,528 INFO [Time-limited test {}] server.Server(415): Started @4507ms 2024-11-25T12:45:52,586 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T12:45:52,727 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:45:52,733 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:45:52,736 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:45:52,736 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:45:52,736 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T12:45:52,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:45:52,738 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:45:52,858 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4464725f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/java.io.tmpdir/jetty-localhost-45007-hadoop-hdfs-3_4_1-tests_jar-_-any-9347151623502151058/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:45:52,859 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:45007} 2024-11-25T12:45:52,860 INFO [Time-limited test {}] server.Server(415): Started @4839ms 2024-11-25T12:45:52,863 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T12:45:52,929 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:45:52,936 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:45:52,948 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:45:52,948 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:45:52,948 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T12:45:52,952 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:45:52,953 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:45:53,070 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26f8cb79{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/java.io.tmpdir/jetty-localhost-33333-hadoop-hdfs-3_4_1-tests_jar-_-any-13841241675672531503/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:45:53,072 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:33333} 2024-11-25T12:45:53,072 INFO [Time-limited test {}] server.Server(415): Started @5051ms 2024-11-25T12:45:53,075 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T12:45:54,143 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data1/current/BP-225838851-172.17.0.3-1732538750914/current, will proceed with Du for space computation calculation, 2024-11-25T12:45:54,143 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data3/current/BP-225838851-172.17.0.3-1732538750914/current, will proceed with Du for space computation calculation, 2024-11-25T12:45:54,143 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data2/current/BP-225838851-172.17.0.3-1732538750914/current, will proceed with Du for space computation calculation, 2024-11-25T12:45:54,143 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data4/current/BP-225838851-172.17.0.3-1732538750914/current, will proceed with Du for space computation calculation, 2024-11-25T12:45:54,211 WARN [Thread-133 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data5/current/BP-225838851-172.17.0.3-1732538750914/current, will proceed with Du for space computation calculation, 2024-11-25T12:45:54,215 WARN [Thread-135 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data6/current/BP-225838851-172.17.0.3-1732538750914/current, will proceed with Du for space computation calculation, 2024-11-25T12:45:54,239 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T12:45:54,240 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T12:45:54,267 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T12:45:54,310 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d8b15c307bc9723 with lease ID 0x64b982d06174aac4: Processing first storage report for DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5 from datanode DatanodeRegistration(127.0.0.1:34959, datanodeUuid=d469b2b5-b085-42d7-aec4-85deee27acef, infoPort=33547, infoSecurePort=0, ipcPort=44747, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914) 2024-11-25T12:45:54,311 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d8b15c307bc9723 with lease ID 0x64b982d06174aac4: from storage DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5 node DatanodeRegistration(127.0.0.1:34959, datanodeUuid=d469b2b5-b085-42d7-aec4-85deee27acef, infoPort=33547, infoSecurePort=0, ipcPort=44747, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-25T12:45:54,312 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa78a0a8d2b9a7aef with lease ID 0x64b982d06174aac2: Processing first storage report for DS-6f16a029-d49d-4918-a940-498eea8876d4 from datanode DatanodeRegistration(127.0.0.1:35653, datanodeUuid=d6dd8a94-3437-49ae-b57b-725cdaf69376, infoPort=43987, infoSecurePort=0, ipcPort=40159, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914) 2024-11-25T12:45:54,312 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa78a0a8d2b9a7aef with lease ID 0x64b982d06174aac2: from storage DS-6f16a029-d49d-4918-a940-498eea8876d4 node DatanodeRegistration(127.0.0.1:35653, datanodeUuid=d6dd8a94-3437-49ae-b57b-725cdaf69376, infoPort=43987, infoSecurePort=0, ipcPort=40159, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T12:45:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x42c465c274b1ac77 with lease ID 0x64b982d06174aac3: Processing first storage report for DS-360fdf6d-0499-4d4c-bea9-178df09bb758 from datanode DatanodeRegistration(127.0.0.1:33377, datanodeUuid=14b2755e-8265-4540-94ea-bca82d164c4d, infoPort=40413, infoSecurePort=0, ipcPort=40517, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914) 2024-11-25T12:45:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x42c465c274b1ac77 with lease ID 0x64b982d06174aac3: from storage DS-360fdf6d-0499-4d4c-bea9-178df09bb758 node DatanodeRegistration(127.0.0.1:33377, datanodeUuid=14b2755e-8265-4540-94ea-bca82d164c4d, infoPort=40413, infoSecurePort=0, ipcPort=40517, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:45:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x42c465c274b1ac77 with lease ID 0x64b982d06174aac3: Processing first storage report for DS-7963cfdb-7bfd-49b9-8458-2a89fdb31d8a from datanode DatanodeRegistration(127.0.0.1:33377, datanodeUuid=14b2755e-8265-4540-94ea-bca82d164c4d, infoPort=40413, infoSecurePort=0, ipcPort=40517, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914) 2024-11-25T12:45:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x42c465c274b1ac77 with lease ID 0x64b982d06174aac3: from storage DS-7963cfdb-7bfd-49b9-8458-2a89fdb31d8a node DatanodeRegistration(127.0.0.1:33377, datanodeUuid=14b2755e-8265-4540-94ea-bca82d164c4d, infoPort=40413, infoSecurePort=0, ipcPort=40517, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:45:54,316 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa78a0a8d2b9a7aef with lease ID 0x64b982d06174aac2: Processing first storage report for DS-0491b021-4d99-4a7f-acc8-c0efea5927e4 from datanode DatanodeRegistration(127.0.0.1:35653, datanodeUuid=d6dd8a94-3437-49ae-b57b-725cdaf69376, infoPort=43987, infoSecurePort=0, ipcPort=40159, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914) 2024-11-25T12:45:54,316 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa78a0a8d2b9a7aef with lease ID 0x64b982d06174aac2: from storage DS-0491b021-4d99-4a7f-acc8-c0efea5927e4 node DatanodeRegistration(127.0.0.1:35653, datanodeUuid=d6dd8a94-3437-49ae-b57b-725cdaf69376, infoPort=43987, infoSecurePort=0, ipcPort=40159, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:45:54,323 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d8b15c307bc9723 with lease ID 0x64b982d06174aac4: Processing first storage report for DS-dbce312f-80b6-494e-9298-46087d851601 from datanode DatanodeRegistration(127.0.0.1:34959, datanodeUuid=d469b2b5-b085-42d7-aec4-85deee27acef, infoPort=33547, infoSecurePort=0, ipcPort=44747, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914) 2024-11-25T12:45:54,324 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d8b15c307bc9723 with lease ID 0x64b982d06174aac4: from storage DS-dbce312f-80b6-494e-9298-46087d851601 node DatanodeRegistration(127.0.0.1:34959, datanodeUuid=d469b2b5-b085-42d7-aec4-85deee27acef, infoPort=33547, infoSecurePort=0, ipcPort=44747, storageInfo=lv=-57;cid=testClusterID;nsid=572246240;c=1732538750914), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:45:54,380 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243 2024-11-25T12:45:54,615 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/zookeeper_0, clientPort=51952, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T12:45:54,694 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51952 2024-11-25T12:45:54,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:54,720 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:55,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741825_1001 (size=7) 2024-11-25T12:45:55,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741825_1001 (size=7) 2024-11-25T12:45:55,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741825_1001 (size=7) 2024-11-25T12:45:55,467 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7 with version=8 2024-11-25T12:45:55,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/hbase-staging 2024-11-25T12:45:55,920 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:45:55,929 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:55,929 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:55,934 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:45:55,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:55,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:45:56,072 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T12:45:56,132 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T12:45:56,145 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T12:45:56,148 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:45:56,173 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 41838 (auto-detected) 2024-11-25T12:45:56,174 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-25T12:45:56,196 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36927 2024-11-25T12:45:56,226 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36927 connecting to ZooKeeper ensemble=127.0.0.1:51952 2024-11-25T12:45:56,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369270x0, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:45:56,332 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36927-0x101735117c30000 connected 2024-11-25T12:45:56,475 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:56,479 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:56,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:56,496 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7, hbase.cluster.distributed=false 2024-11-25T12:45:56,527 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:45:56,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36927 2024-11-25T12:45:56,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36927 2024-11-25T12:45:56,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36927 2024-11-25T12:45:56,548 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36927 2024-11-25T12:45:56,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36927 2024-11-25T12:45:56,700 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:45:56,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:56,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:56,704 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:45:56,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:56,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:45:56,709 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T12:45:56,716 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:45:56,719 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43295 2024-11-25T12:45:56,722 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43295 connecting to ZooKeeper ensemble=127.0.0.1:51952 2024-11-25T12:45:56,724 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:56,731 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:56,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432950x0, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:45:56,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43295-0x101735117c30001 connected 2024-11-25T12:45:56,764 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:56,773 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T12:45:56,788 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T12:45:56,792 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T12:45:56,802 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:45:56,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43295 2024-11-25T12:45:56,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43295 2024-11-25T12:45:56,809 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43295 2024-11-25T12:45:56,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43295 2024-11-25T12:45:56,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43295 2024-11-25T12:45:56,841 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:45:56,842 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:56,842 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:56,843 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:45:56,844 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:56,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:45:56,866 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T12:45:56,867 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:45:56,880 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38825 2024-11-25T12:45:56,883 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38825 connecting to ZooKeeper ensemble=127.0.0.1:51952 2024-11-25T12:45:56,884 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:56,890 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:56,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388250x0, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:45:56,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388250x0, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:56,955 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T12:45:56,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38825-0x101735117c30002 connected 2024-11-25T12:45:56,976 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T12:45:56,980 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T12:45:56,988 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:45:56,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38825 2024-11-25T12:45:56,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38825 2024-11-25T12:45:56,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38825 2024-11-25T12:45:57,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38825 2024-11-25T12:45:57,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38825 2024-11-25T12:45:57,029 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:45:57,029 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:57,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:57,030 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:45:57,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:57,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:45:57,031 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T12:45:57,031 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:45:57,032 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35205 2024-11-25T12:45:57,035 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35205 connecting to ZooKeeper ensemble=127.0.0.1:51952 2024-11-25T12:45:57,037 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:57,041 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:57,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352050x0, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:45:57,057 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:352050x0, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:57,057 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35205-0x101735117c30003 connected 2024-11-25T12:45:57,058 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T12:45:57,059 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T12:45:57,061 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T12:45:57,063 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:45:57,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35205 2024-11-25T12:45:57,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35205 2024-11-25T12:45:57,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35205 2024-11-25T12:45:57,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35205 2024-11-25T12:45:57,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35205 2024-11-25T12:45:57,083 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1ff8009e463e:36927 2024-11-25T12:45:57,084 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1ff8009e463e,36927,1732538755691 2024-11-25T12:45:57,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:57,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:57,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:57,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:57,104 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1ff8009e463e,36927,1732538755691 2024-11-25T12:45:57,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T12:45:57,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T12:45:57,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T12:45:57,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:57,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:57,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:57,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:57,149 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T12:45:57,150 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1ff8009e463e,36927,1732538755691 from backup master directory 2024-11-25T12:45:57,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:57,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1ff8009e463e,36927,1732538755691 2024-11-25T12:45:57,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:57,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:57,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:57,193 WARN [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:45:57,194 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1ff8009e463e,36927,1732538755691 2024-11-25T12:45:57,196 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T12:45:57,198 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T12:45:57,253 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/hbase.id] with ID: c2a5e957-eae0-4d39-83b1-2894938e9454 2024-11-25T12:45:57,254 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/.tmp/hbase.id 2024-11-25T12:45:57,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741826_1002 (size=42) 2024-11-25T12:45:57,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741826_1002 (size=42) 2024-11-25T12:45:57,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741826_1002 (size=42) 2024-11-25T12:45:57,270 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/.tmp/hbase.id]:[hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/hbase.id] 2024-11-25T12:45:57,316 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:57,321 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T12:45:57,343 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-25T12:45:57,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:57,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:57,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:57,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:57,398 WARN [IPC Server handler 4 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:57,398 WARN [IPC Server handler 4 on default port 35793 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:57,399 WARN [IPC Server handler 4 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:57,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741827_1003 (size=196) 2024-11-25T12:45:57,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741827_1003 (size=196) 2024-11-25T12:45:57,443 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:45:57,446 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T12:45:57,452 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:57,493 WARN [IPC Server handler 1 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:57,493 WARN [IPC Server handler 1 on default port 35793 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:57,494 WARN [IPC Server handler 1 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:57,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741828_1004 (size=1189) 2024-11-25T12:45:57,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741828_1004 (size=1189) 2024-11-25T12:45:57,532 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store 2024-11-25T12:45:57,560 WARN [IPC Server handler 4 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:57,561 WARN [IPC Server handler 4 on default port 35793 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:57,561 WARN [IPC Server handler 4 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:57,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741829_1005 (size=34) 2024-11-25T12:45:57,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741829_1005 (size=34) 2024-11-25T12:45:57,590 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T12:45:57,594 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:57,596 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T12:45:57,596 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:57,596 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:57,598 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T12:45:57,598 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:57,598 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:57,600 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732538757595Disabling compacts and flushes for region at 1732538757595Disabling writes for close at 1732538757598 (+3 ms)Writing region close event to WAL at 1732538757598Closed at 1732538757598 2024-11-25T12:45:57,603 WARN [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/.initializing 2024-11-25T12:45:57,603 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/WALs/1ff8009e463e,36927,1732538755691 2024-11-25T12:45:57,612 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:57,629 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C36927%2C1732538755691, suffix=, logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/WALs/1ff8009e463e,36927,1732538755691, archiveDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/oldWALs, maxLogs=10 2024-11-25T12:45:57,669 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/WALs/1ff8009e463e,36927,1732538755691/1ff8009e463e%2C36927%2C1732538755691.1732538757635, exclude list is [], retry=0 2024-11-25T12:45:57,673 WARN [IPC Server handler 3 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:57,673 WARN [IPC Server handler 3 on default port 35793 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:57,673 WARN [IPC Server handler 3 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:57,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:57,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:45:57,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:45:57,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T12:45:57,750 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/WALs/1ff8009e463e,36927,1732538755691/1ff8009e463e%2C36927%2C1732538755691.1732538757635 2024-11-25T12:45:57,751 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:45:57,751 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:57,752 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:57,755 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,756 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,803 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,840 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T12:45:57,845 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:57,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:57,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,855 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T12:45:57,855 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:57,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:57,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T12:45:57,864 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:57,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:57,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T12:45:57,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:57,872 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:57,872 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,877 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,879 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,887 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,888 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,893 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:45:57,905 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:57,924 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:57,928 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69685022, jitterRate=0.03838774561882019}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:45:57,941 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732538757767Initializing all the Stores at 1732538757769 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538757770 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538757770Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538757771 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538757771Cleaning up temporary data from old regions at 1732538757888 (+117 ms)Region opened successfully at 1732538757941 (+53 ms) 2024-11-25T12:45:57,951 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T12:45:58,021 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9bcdfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:45:58,066 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T12:45:58,078 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T12:45:58,079 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T12:45:58,083 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T12:45:58,095 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 11 msec 2024-11-25T12:45:58,102 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-25T12:45:58,104 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T12:45:58,155 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T12:45:58,172 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T12:45:58,210 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T12:45:58,213 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T12:45:58,216 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T12:45:58,234 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T12:45:58,237 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T12:45:58,246 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T12:45:58,255 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T12:45:58,258 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T12:45:58,268 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T12:45:58,299 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:58,299 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:58,299 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:58,300 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:58,300 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:58,300 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:58,300 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:58,301 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:58,301 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:58,301 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:58,301 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:58,302 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:58,302 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T12:45:58,332 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T12:45:58,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:58,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:58,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:58,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:58,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,401 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1ff8009e463e,36927,1732538755691, sessionid=0x101735117c30000, setting cluster-up flag (Was=false) 2024-11-25T12:45:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,637 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T12:45:58,641 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ff8009e463e,36927,1732538755691 2024-11-25T12:45:58,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:58,967 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T12:45:58,971 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ff8009e463e,36927,1732538755691 2024-11-25T12:45:58,989 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T12:45:59,094 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(746): ClusterId : c2a5e957-eae0-4d39-83b1-2894938e9454 2024-11-25T12:45:59,094 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(746): ClusterId : c2a5e957-eae0-4d39-83b1-2894938e9454 2024-11-25T12:45:59,096 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(746): ClusterId : c2a5e957-eae0-4d39-83b1-2894938e9454 2024-11-25T12:45:59,098 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T12:45:59,099 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T12:45:59,100 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T12:45:59,117 INFO [AsyncFSWAL-0-hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData-prefix:1ff8009e463e,36927,1732538755691 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-25T12:45:59,136 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T12:45:59,150 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T12:45:59,159 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T12:45:59,167 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1ff8009e463e,36927,1732538755691 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T12:45:59,180 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T12:45:59,181 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T12:45:59,185 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T12:45:59,185 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T12:45:59,185 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T12:45:59,186 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T12:45:59,273 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:45:59,273 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:45:59,273 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:45:59,274 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:45:59,274 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1ff8009e463e:0, corePoolSize=10, maxPoolSize=10 2024-11-25T12:45:59,274 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,274 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:45:59,275 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,276 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732538789276 2024-11-25T12:45:59,278 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T12:45:59,279 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T12:45:59,283 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T12:45:59,283 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T12:45:59,284 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T12:45:59,284 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T12:45:59,285 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,288 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:45:59,289 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T12:45:59,290 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T12:45:59,291 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T12:45:59,292 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T12:45:59,295 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:59,295 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T12:45:59,296 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T12:45:59,295 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T12:45:59,299 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732538759297,5,FailOnTimeoutGroup] 2024-11-25T12:45:59,303 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732538759299,5,FailOnTimeoutGroup] 2024-11-25T12:45:59,304 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,304 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T12:45:59,305 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,305 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,307 WARN [IPC Server handler 3 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:59,307 WARN [IPC Server handler 3 on default port 35793 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:59,307 WARN [IPC Server handler 3 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:59,323 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T12:45:59,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741831_1007 (size=1321) 2024-11-25T12:45:59,323 DEBUG [RS:2;1ff8009e463e:35205 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d8a39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:45:59,323 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T12:45:59,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741831_1007 (size=1321) 2024-11-25T12:45:59,324 DEBUG [RS:1;1ff8009e463e:38825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1384f9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:45:59,325 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T12:45:59,325 DEBUG [RS:0;1ff8009e463e:43295 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b2ceea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:45:59,345 DEBUG [RS:0;1ff8009e463e:43295 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1ff8009e463e:43295 2024-11-25T12:45:59,345 DEBUG [RS:1;1ff8009e463e:38825 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1ff8009e463e:38825 2024-11-25T12:45:59,345 DEBUG [RS:2;1ff8009e463e:35205 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1ff8009e463e:35205 2024-11-25T12:45:59,350 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T12:45:59,350 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T12:45:59,350 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T12:45:59,350 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T12:45:59,350 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T12:45:59,350 DEBUG [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T12:45:59,350 DEBUG [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T12:45:59,350 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T12:45:59,350 DEBUG [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T12:45:59,353 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,36927,1732538755691 with port=43295, startcode=1732538756657 2024-11-25T12:45:59,354 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,36927,1732538755691 with port=35205, startcode=1732538757028 2024-11-25T12:45:59,356 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,36927,1732538755691 with port=38825, startcode=1732538756840 2024-11-25T12:45:59,368 DEBUG [RS:2;1ff8009e463e:35205 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T12:45:59,368 DEBUG [RS:0;1ff8009e463e:43295 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T12:45:59,369 DEBUG [RS:1;1ff8009e463e:38825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T12:45:59,410 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55639, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T12:45:59,410 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56879, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T12:45:59,410 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T12:45:59,416 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36927 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ff8009e463e,38825,1732538756840 2024-11-25T12:45:59,419 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36927 {}] master.ServerManager(517): Registering regionserver=1ff8009e463e,38825,1732538756840 2024-11-25T12:45:59,430 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36927 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ff8009e463e,43295,1732538756657 2024-11-25T12:45:59,430 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36927 {}] master.ServerManager(517): Registering regionserver=1ff8009e463e,43295,1732538756657 2024-11-25T12:45:59,434 DEBUG [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7 2024-11-25T12:45:59,434 DEBUG [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35793 2024-11-25T12:45:59,434 DEBUG [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T12:45:59,435 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36927 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ff8009e463e,35205,1732538757028 2024-11-25T12:45:59,436 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36927 {}] master.ServerManager(517): Registering regionserver=1ff8009e463e,35205,1732538757028 2024-11-25T12:45:59,439 DEBUG [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7 2024-11-25T12:45:59,439 DEBUG [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35793 2024-11-25T12:45:59,439 DEBUG [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T12:45:59,440 DEBUG [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7 2024-11-25T12:45:59,440 DEBUG [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35793 2024-11-25T12:45:59,440 DEBUG [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T12:45:59,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:45:59,580 DEBUG [RS:1;1ff8009e463e:38825 {}] zookeeper.ZKUtil(111): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ff8009e463e,38825,1732538756840 2024-11-25T12:45:59,580 WARN [RS:1;1ff8009e463e:38825 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:45:59,580 INFO [RS:1;1ff8009e463e:38825 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:59,581 DEBUG [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840 2024-11-25T12:45:59,582 DEBUG [RS:2;1ff8009e463e:35205 {}] zookeeper.ZKUtil(111): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ff8009e463e,35205,1732538757028 2024-11-25T12:45:59,582 DEBUG [RS:0;1ff8009e463e:43295 {}] zookeeper.ZKUtil(111): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ff8009e463e,43295,1732538756657 2024-11-25T12:45:59,582 WARN [RS:2;1ff8009e463e:35205 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:45:59,582 WARN [RS:0;1ff8009e463e:43295 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:45:59,582 INFO [RS:2;1ff8009e463e:35205 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:59,582 INFO [RS:0;1ff8009e463e:43295 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:59,582 DEBUG [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,35205,1732538757028 2024-11-25T12:45:59,582 DEBUG [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,43295,1732538756657 2024-11-25T12:45:59,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ff8009e463e,38825,1732538756840] 2024-11-25T12:45:59,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ff8009e463e,43295,1732538756657] 2024-11-25T12:45:59,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ff8009e463e,35205,1732538757028] 2024-11-25T12:45:59,616 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T12:45:59,616 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T12:45:59,616 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T12:45:59,640 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T12:45:59,657 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T12:45:59,658 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T12:45:59,663 INFO [RS:1;1ff8009e463e:38825 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:45:59,663 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,664 INFO [RS:0;1ff8009e463e:43295 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:45:59,664 INFO [RS:2;1ff8009e463e:35205 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:45:59,664 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,664 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,669 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T12:45:59,671 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T12:45:59,673 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T12:45:59,678 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T12:45:59,679 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T12:45:59,680 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T12:45:59,681 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,681 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,681 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,681 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,681 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,681 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,682 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:45:59,682 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:45:59,682 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,683 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,683 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,683 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,683 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,683 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,683 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,683 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:59,683 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,683 DEBUG [RS:0;1ff8009e463e:43295 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:59,684 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,684 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,684 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,684 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,684 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:59,684 DEBUG [RS:1;1ff8009e463e:38825 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:59,684 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,685 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,685 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:45:59,685 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,685 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,685 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,685 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,686 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,686 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:59,686 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:59,686 DEBUG [RS:2;1ff8009e463e:35205 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:59,707 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,708 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,708 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,708 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,708 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,708 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,43295,1732538756657-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:45:59,713 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,713 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,714 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,714 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,714 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,714 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,38825,1732538756840-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:45:59,722 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,722 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,722 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,722 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,723 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,723 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,35205,1732538757028-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:45:59,727 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T12:45:59,728 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7 2024-11-25T12:45:59,742 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T12:45:59,745 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T12:45:59,745 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,43295,1732538756657-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,746 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,746 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.Replication(171): 1ff8009e463e,43295,1732538756657 started 2024-11-25T12:45:59,749 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,38825,1732538756840-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,750 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,750 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.Replication(171): 1ff8009e463e,38825,1732538756840 started 2024-11-25T12:45:59,766 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T12:45:59,766 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,35205,1732538757028-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,766 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,766 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.Replication(171): 1ff8009e463e,35205,1732538757028 started 2024-11-25T12:45:59,772 WARN [IPC Server handler 0 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:59,772 WARN [IPC Server handler 0 on default port 35793 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:59,773 WARN [IPC Server handler 0 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:59,779 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,781 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,785 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1482): Serving as 1ff8009e463e,43295,1732538756657, RpcServer on 1ff8009e463e/172.17.0.3:43295, sessionid=0x101735117c30001 2024-11-25T12:45:59,785 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1482): Serving as 1ff8009e463e,38825,1732538756840, RpcServer on 1ff8009e463e/172.17.0.3:38825, sessionid=0x101735117c30002 2024-11-25T12:45:59,786 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T12:45:59,786 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T12:45:59,787 DEBUG [RS:0;1ff8009e463e:43295 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ff8009e463e,43295,1732538756657 2024-11-25T12:45:59,787 DEBUG [RS:1;1ff8009e463e:38825 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ff8009e463e,38825,1732538756840 2024-11-25T12:45:59,787 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,38825,1732538756840' 2024-11-25T12:45:59,787 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,43295,1732538756657' 2024-11-25T12:45:59,787 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T12:45:59,787 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T12:45:59,788 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T12:45:59,788 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T12:45:59,789 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T12:45:59,789 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T12:45:59,790 DEBUG [RS:0;1ff8009e463e:43295 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ff8009e463e,43295,1732538756657 2024-11-25T12:45:59,790 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T12:45:59,790 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T12:45:59,790 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,43295,1732538756657' 2024-11-25T12:45:59,790 DEBUG [RS:1;1ff8009e463e:38825 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ff8009e463e,38825,1732538756840 2024-11-25T12:45:59,790 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T12:45:59,790 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,38825,1732538756840' 2024-11-25T12:45:59,790 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T12:45:59,791 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T12:45:59,791 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T12:45:59,792 DEBUG [RS:1;1ff8009e463e:38825 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T12:45:59,792 DEBUG [RS:0;1ff8009e463e:43295 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T12:45:59,792 INFO [RS:1;1ff8009e463e:38825 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T12:45:59,792 INFO [RS:0;1ff8009e463e:43295 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T12:45:59,792 INFO [RS:0;1ff8009e463e:43295 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T12:45:59,792 INFO [RS:1;1ff8009e463e:38825 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T12:45:59,797 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:59,798 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1482): Serving as 1ff8009e463e,35205,1732538757028, RpcServer on 1ff8009e463e/172.17.0.3:35205, sessionid=0x101735117c30003 2024-11-25T12:45:59,798 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T12:45:59,798 DEBUG [RS:2;1ff8009e463e:35205 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ff8009e463e,35205,1732538757028 2024-11-25T12:45:59,798 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,35205,1732538757028' 2024-11-25T12:45:59,798 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T12:45:59,800 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T12:45:59,801 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T12:45:59,802 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T12:45:59,802 DEBUG [RS:2;1ff8009e463e:35205 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ff8009e463e,35205,1732538757028 2024-11-25T12:45:59,802 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,35205,1732538757028' 2024-11-25T12:45:59,802 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T12:45:59,803 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T12:45:59,804 DEBUG [RS:2;1ff8009e463e:35205 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T12:45:59,804 INFO [RS:2;1ff8009e463e:35205 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T12:45:59,804 INFO [RS:2;1ff8009e463e:35205 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T12:45:59,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741832_1008 (size=32) 2024-11-25T12:45:59,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741832_1008 (size=32) 2024-11-25T12:45:59,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:59,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:45:59,822 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:45:59,822 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:59,824 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:59,824 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:45:59,828 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:45:59,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:59,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:59,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:45:59,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:45:59,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:59,847 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:59,847 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:45:59,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:45:59,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:59,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:59,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:45:59,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740 2024-11-25T12:45:59,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740 2024-11-25T12:45:59,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:45:59,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:45:59,886 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:45:59,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:45:59,898 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:59,900 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72078481, jitterRate=0.07405306398868561}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:45:59,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732538759815Initializing all the Stores at 1732538759817 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538759817Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538759818 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538759818Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538759818Cleaning up temporary data from old regions at 1732538759884 (+66 ms)Region opened successfully at 1732538759901 (+17 ms) 2024-11-25T12:45:59,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T12:45:59,902 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T12:45:59,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T12:45:59,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T12:45:59,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T12:45:59,904 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T12:45:59,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732538759902Disabling compacts and flushes for region at 1732538759902Disabling writes for close at 1732538759902Writing region close event to WAL at 1732538759903 (+1 ms)Closed at 1732538759904 (+1 ms) 2024-11-25T12:45:59,909 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:45:59,909 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T12:45:59,914 INFO [RS:2;1ff8009e463e:35205 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:59,915 INFO [RS:1;1ff8009e463e:38825 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:59,917 INFO [RS:0;1ff8009e463e:43295 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:59,921 INFO [RS:1;1ff8009e463e:38825 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C38825%2C1732538756840, suffix=, logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840, archiveDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs, maxLogs=32 2024-11-25T12:45:59,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T12:45:59,934 INFO [RS:0;1ff8009e463e:43295 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C43295%2C1732538756657, suffix=, logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,43295,1732538756657, archiveDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs, maxLogs=32 2024-11-25T12:45:59,939 INFO [RS:2;1ff8009e463e:35205 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C35205%2C1732538757028, suffix=, logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,35205,1732538757028, archiveDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs, maxLogs=32 2024-11-25T12:45:59,970 DEBUG [RS:0;1ff8009e463e:43295 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,43295,1732538756657/1ff8009e463e%2C43295%2C1732538756657.1732538759942, exclude list is [], retry=0 2024-11-25T12:45:59,970 DEBUG [RS:2;1ff8009e463e:35205 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,35205,1732538757028/1ff8009e463e%2C35205%2C1732538757028.1732538759942, exclude list is [], retry=0 2024-11-25T12:45:59,971 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:45:59,973 DEBUG [RS:1;1ff8009e463e:38825 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840/1ff8009e463e%2C38825%2C1732538756840.1732538759929, exclude list is [], retry=0 2024-11-25T12:45:59,985 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T12:45:59,987 WARN [IPC Server handler 1 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:59,987 WARN [IPC Server handler 1 on default port 35793 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:59,987 WARN [IPC Server handler 1 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:59,987 WARN [IPC Server handler 3 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-25T12:45:59,988 WARN [IPC Server handler 3 on default port 35793 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-25T12:45:59,988 WARN [IPC Server handler 3 on default port 35793 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-25T12:45:59,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:45:59,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:45:59,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:45:59,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:45:59,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:45:59,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:45:59,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:00,043 INFO [RS:0;1ff8009e463e:43295 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,43295,1732538756657/1ff8009e463e%2C43295%2C1732538756657.1732538759942 2024-11-25T12:46:00,057 INFO [RS:2;1ff8009e463e:35205 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,35205,1732538757028/1ff8009e463e%2C35205%2C1732538757028.1732538759942 2024-11-25T12:46:00,058 DEBUG [RS:0;1ff8009e463e:43295 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:00,067 INFO [RS:1;1ff8009e463e:38825 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840/1ff8009e463e%2C38825%2C1732538756840.1732538759929 2024-11-25T12:46:00,071 DEBUG [RS:2;1ff8009e463e:35205 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:00,078 DEBUG [RS:1;1ff8009e463e:38825 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:00,140 DEBUG [1ff8009e463e:36927 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-25T12:46:00,152 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(204): Hosts are {1ff8009e463e=0} racks are {/default-rack=0} 2024-11-25T12:46:00,163 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T12:46:00,164 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T12:46:00,164 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T12:46:00,164 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T12:46:00,164 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T12:46:00,164 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T12:46:00,164 INFO [1ff8009e463e:36927 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T12:46:00,164 INFO [1ff8009e463e:36927 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T12:46:00,164 INFO [1ff8009e463e:36927 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T12:46:00,164 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T12:46:00,174 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ff8009e463e,38825,1732538756840 2024-11-25T12:46:00,181 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ff8009e463e,38825,1732538756840, state=OPENING 2024-11-25T12:46:00,205 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T12:46:00,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:46:00,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:46:00,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:46:00,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:46:00,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:00,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:00,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:00,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:00,219 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:46:00,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ff8009e463e,38825,1732538756840}] 2024-11-25T12:46:00,410 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:46:00,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42819, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:46:00,451 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T12:46:00,452 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:00,453 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T12:46:00,460 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C38825%2C1732538756840.meta, suffix=.meta, logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840, archiveDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs, maxLogs=32 2024-11-25T12:46:00,484 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, exclude list is [], retry=0 2024-11-25T12:46:00,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:00,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:00,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:00,507 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta 2024-11-25T12:46:00,508 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:00,509 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:00,511 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T12:46:00,515 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T12:46:00,523 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T12:46:00,529 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T12:46:00,530 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:00,531 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T12:46:00,531 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T12:46:00,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:46:00,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:46:00,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:00,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:46:00,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:46:00,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:46:00,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:00,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:46:00,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:46:00,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:46:00,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:00,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:46:00,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:46:00,546 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:46:00,546 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:00,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:46:00,547 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:46:00,549 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740 2024-11-25T12:46:00,553 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740 2024-11-25T12:46:00,555 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:46:00,556 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:46:00,557 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:46:00,561 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:46:00,563 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69067521, jitterRate=0.02918626368045807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:46:00,564 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T12:46:00,566 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732538760531Writing region info on filesystem at 1732538760532 (+1 ms)Initializing all the Stores at 1732538760534 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538760534Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538760534Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538760534Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538760534Cleaning up temporary data from old regions at 1732538760556 (+22 ms)Running coprocessor post-open hooks at 1732538760564 (+8 ms)Region opened successfully at 1732538760566 (+2 ms) 2024-11-25T12:46:00,574 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732538760397 2024-11-25T12:46:00,588 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T12:46:00,588 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T12:46:00,590 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,38825,1732538756840 2024-11-25T12:46:00,593 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ff8009e463e,38825,1732538756840, state=OPEN 2024-11-25T12:46:00,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:00,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:00,612 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:00,612 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:00,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:00,612 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:00,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:00,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:00,614 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ff8009e463e,38825,1732538756840 2024-11-25T12:46:00,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T12:46:00,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ff8009e463e,38825,1732538756840 in 393 msec 2024-11-25T12:46:00,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T12:46:00,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 711 msec 2024-11-25T12:46:00,643 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:46:00,643 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T12:46:00,666 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:46:00,668 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1] 2024-11-25T12:46:00,688 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:46:00,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40961, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:46:00,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6800 sec 2024-11-25T12:46:00,720 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732538760720, completionTime=-1 2024-11-25T12:46:00,724 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-25T12:46:00,724 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T12:46:00,779 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-25T12:46:00,779 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732538820779 2024-11-25T12:46:00,780 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732538880780 2024-11-25T12:46:00,780 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 55 msec 2024-11-25T12:46:00,782 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T12:46:00,791 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,36927,1732538755691-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:00,791 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,36927,1732538755691-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:00,791 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,36927,1732538755691-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:00,793 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1ff8009e463e:36927, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:00,794 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:00,795 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:00,812 DEBUG [master/1ff8009e463e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T12:46:00,837 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.643sec 2024-11-25T12:46:00,839 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T12:46:00,841 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T12:46:00,842 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T12:46:00,843 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T12:46:00,843 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T12:46:00,844 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,36927,1732538755691-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:46:00,844 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,36927,1732538755691-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T12:46:00,850 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T12:46:00,851 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T12:46:00,852 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,36927,1732538755691-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:00,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5526c77c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:46:00,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,36927,-1 for getting cluster id 2024-11-25T12:46:00,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:46:00,928 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c2a5e957-eae0-4d39-83b1-2894938e9454' 2024-11-25T12:46:00,930 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:46:00,930 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c2a5e957-eae0-4d39-83b1-2894938e9454" 2024-11-25T12:46:00,931 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67469cfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:46:00,931 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,36927,-1] 2024-11-25T12:46:00,933 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:46:00,937 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:00,938 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:46:00,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bcc98ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:46:00,942 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:46:00,955 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1] 2024-11-25T12:46:00,956 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:46:00,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:46:00,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1ff8009e463e,36927,1732538755691 2024-11-25T12:46:00,986 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35793/hbase 2024-11-25T12:46:01,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=356, OpenFileDescriptor=589, MaxFileDescriptor=1048576, SystemLoadAverage=620, ProcessCount=11, AvailableMemoryMB=2771 2024-11-25T12:46:01,120 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:01,131 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:01,136 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:01,170 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15036654, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-15036654, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:01,204 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15036654/hregion-15036654.1732538761171, exclude list is [], retry=0 2024-11-25T12:46:01,225 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:01,230 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:01,233 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:01,296 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15036654/hregion-15036654.1732538761171 2024-11-25T12:46:01,319 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:01,320 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7de63724b1c8a4cac3688d11a22b57fd, NAME => 'testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:01,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741838_1014 (size=64) 2024-11-25T12:46:01,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741838_1014 (size=64) 2024-11-25T12:46:01,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741838_1014 (size=64) 2024-11-25T12:46:01,389 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:01,401 INFO [StoreOpener-7de63724b1c8a4cac3688d11a22b57fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,415 INFO [StoreOpener-7de63724b1c8a4cac3688d11a22b57fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7de63724b1c8a4cac3688d11a22b57fd columnFamilyName a 2024-11-25T12:46:01,415 DEBUG [StoreOpener-7de63724b1c8a4cac3688d11a22b57fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:01,416 INFO [StoreOpener-7de63724b1c8a4cac3688d11a22b57fd-1 {}] regionserver.HStore(327): Store=7de63724b1c8a4cac3688d11a22b57fd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:01,417 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,418 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,420 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,425 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,425 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,430 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,436 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:01,437 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7de63724b1c8a4cac3688d11a22b57fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75169472, jitterRate=0.12011241912841797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:01,438 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7de63724b1c8a4cac3688d11a22b57fd: Writing region info on filesystem at 1732538761389Initializing all the Stores at 1732538761398 (+9 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538761398Cleaning up temporary data from old regions at 1732538761425 (+27 ms)Region opened successfully at 1732538761438 (+13 ms) 2024-11-25T12:46:01,438 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7de63724b1c8a4cac3688d11a22b57fd, disabling compactions & flushes 2024-11-25T12:46:01,438 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd. 2024-11-25T12:46:01,438 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd. 2024-11-25T12:46:01,439 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd. after waiting 0 ms 2024-11-25T12:46:01,439 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd. 2024-11-25T12:46:01,439 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd. 2024-11-25T12:46:01,439 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7de63724b1c8a4cac3688d11a22b57fd: Waiting for close lock at 1732538761438Disabling compacts and flushes for region at 1732538761438Disabling writes for close at 1732538761439 (+1 ms)Writing region close event to WAL at 1732538761439Closed at 1732538761439 2024-11-25T12:46:01,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741837_1013 (size=95) 2024-11-25T12:46:01,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741837_1013 (size=95) 2024-11-25T12:46:01,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741837_1013 (size=95) 2024-11-25T12:46:01,453 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:01,453 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15036654:(num 1732538761171) 2024-11-25T12:46:01,455 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T12:46:01,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741839_1015 (size=320) 2024-11-25T12:46:01,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741839_1015 (size=320) 2024-11-25T12:46:01,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741839_1015 (size=320) 2024-11-25T12:46:01,486 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T12:46:01,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741840_1016 (size=253) 2024-11-25T12:46:01,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741840_1016 (size=253) 2024-11-25T12:46:01,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741840_1016 (size=253) 2024-11-25T12:46:01,533 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1, size=320 (320bytes) 2024-11-25T12:46:01,534 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-25T12:46:01,534 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-25T12:46:01,535 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1 2024-11-25T12:46:01,541 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1 after 3ms 2024-11-25T12:46:01,548 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:01,548 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1 took 17ms 2024-11-25T12:46:01,562 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1 so closing down 2024-11-25T12:46:01,562 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:01,568 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-25T12:46:01,570 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000001-wal-1.temp 2024-11-25T12:46:01,572 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:01,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741841_1017 (size=320) 2024-11-25T12:46:01,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741841_1017 (size=320) 2024-11-25T12:46:01,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741841_1017 (size=320) 2024-11-25T12:46:01,595 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:01,615 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002 2024-11-25T12:46:01,635 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-25T12:46:01,636 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1, journal: Splitting hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1, size=320 (320bytes) at 1732538761533Finishing writing output for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1 so closing down at 1732538761562 (+29 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000001-wal-1.temp at 1732538761570 (+8 ms)3 split writer threads finished at 1732538761572 (+2 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732538761595 (+23 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002 at 1732538761615 (+20 ms)Processed 2 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732538761635 (+20 ms) 2024-11-25T12:46:01,660 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2, size=253 (253bytes) 2024-11-25T12:46:01,661 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2 2024-11-25T12:46:01,665 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2 after 4ms 2024-11-25T12:46:01,677 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:01,677 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2 took 17ms 2024-11-25T12:46:01,686 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2 so closing down 2024-11-25T12:46:01,686 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:01,689 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-25T12:46:01,691 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002-wal-2.temp 2024-11-25T12:46:01,691 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741842_1018 (size=253) 2024-11-25T12:46:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741842_1018 (size=253) 2024-11-25T12:46:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741842_1018 (size=253) 2024-11-25T12:46:01,718 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:01,723 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:01,727 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-25T12:46:01,736 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 52 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-25T12:46:01,736 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2, journal: Splitting hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2, size=253 (253bytes) at 1732538761661Finishing writing output for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2 so closing down at 1732538761686 (+25 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002-wal-2.temp at 1732538761691 (+5 ms)3 split writer threads finished at 1732538761692 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732538761718 (+26 ms)Processed 1 edits across 1 Regions in 52 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732538761736 (+18 ms) 2024-11-25T12:46:01,736 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:01,740 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:01,760 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal.1732538761741, exclude list is [], retry=0 2024-11-25T12:46:01,765 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:01,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:01,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:01,772 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal.1732538761741 2024-11-25T12:46:01,773 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:01,773 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7de63724b1c8a4cac3688d11a22b57fd, NAME => 'testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:01,773 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:01,773 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,773 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,776 INFO [StoreOpener-7de63724b1c8a4cac3688d11a22b57fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,778 INFO [StoreOpener-7de63724b1c8a4cac3688d11a22b57fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7de63724b1c8a4cac3688d11a22b57fd columnFamilyName a 2024-11-25T12:46:01,779 DEBUG [StoreOpener-7de63724b1c8a4cac3688d11a22b57fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:01,780 INFO [StoreOpener-7de63724b1c8a4cac3688d11a22b57fd-1 {}] regionserver.HStore(327): Store=7de63724b1c8a4cac3688d11a22b57fd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:01,780 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,782 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,786 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,788 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002 2024-11-25T12:46:01,794 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:01,801 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002 2024-11-25T12:46:01,806 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7de63724b1c8a4cac3688d11a22b57fd 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-25T12:46:01,872 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/.tmp/a/d38b91e705b44dbf8b1dd167de67c40f is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732538761454/Put/seqid=0 2024-11-25T12:46:01,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741844_1020 (size=5170) 2024-11-25T12:46:01,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741844_1020 (size=5170) 2024-11-25T12:46:01,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741844_1020 (size=5170) 2024-11-25T12:46:01,893 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/.tmp/a/d38b91e705b44dbf8b1dd167de67c40f 2024-11-25T12:46:01,938 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/.tmp/a/d38b91e705b44dbf8b1dd167de67c40f as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/a/d38b91e705b44dbf8b1dd167de67c40f 2024-11-25T12:46:01,948 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/a/d38b91e705b44dbf8b1dd167de67c40f, entries=2, sequenceid=2, filesize=5.0 K 2024-11-25T12:46:01,953 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7de63724b1c8a4cac3688d11a22b57fd in 148ms, sequenceid=2, compaction requested=false; wal=null 2024-11-25T12:46:01,955 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/0000000000000000002 2024-11-25T12:46:01,956 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,956 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,959 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7de63724b1c8a4cac3688d11a22b57fd 2024-11-25T12:46:01,962 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/7de63724b1c8a4cac3688d11a22b57fd/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-25T12:46:01,964 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7de63724b1c8a4cac3688d11a22b57fd; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63531945, jitterRate=-0.053300246596336365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:01,965 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7de63724b1c8a4cac3688d11a22b57fd: Writing region info on filesystem at 1732538761774Initializing all the Stores at 1732538761776 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538761776Obtaining lock to block concurrent updates at 1732538761806 (+30 ms)Preparing flush snapshotting stores in 7de63724b1c8a4cac3688d11a22b57fd at 1732538761806Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732538761811 (+5 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732538761124.7de63724b1c8a4cac3688d11a22b57fd. at 1732538761811Flushing 7de63724b1c8a4cac3688d11a22b57fd/a: creating writer at 1732538761813 (+2 ms)Flushing 7de63724b1c8a4cac3688d11a22b57fd/a: appending metadata at 1732538761859 (+46 ms)Flushing 7de63724b1c8a4cac3688d11a22b57fd/a: closing flushed file at 1732538761862 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37001814: reopening flushed file at 1732538761936 (+74 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7de63724b1c8a4cac3688d11a22b57fd in 148ms, sequenceid=2, compaction requested=false; wal=null at 1732538761953 (+17 ms)Cleaning up temporary data from old regions at 1732538761956 (+3 ms)Region opened successfully at 1732538761965 (+9 ms) 2024-11-25T12:46:01,997 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=368 (was 356) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:39916 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:52976 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35793/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:52948 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:39858 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:41802 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:41828 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=683 (was 589) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=620 (was 620), ProcessCount=11 (was 11), AvailableMemoryMB=2630 (was 2771) 2024-11-25T12:46:02,013 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=368, OpenFileDescriptor=683, MaxFileDescriptor=1048576, SystemLoadAverage=620, ProcessCount=11, AvailableMemoryMB=2624 2024-11-25T12:46:02,029 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:02,033 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:02,034 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:02,038 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-06333675, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-06333675, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:02,052 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-06333675/hregion-06333675.1732538762039, exclude list is [], retry=0 2024-11-25T12:46:02,056 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:02,056 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:02,057 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:02,061 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-06333675/hregion-06333675.1732538762039 2024-11-25T12:46:02,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:02,063 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 23f87995af9a0ade941413c0db58171f, NAME => 'testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741846_1022 (size=64) 2024-11-25T12:46:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741846_1022 (size=64) 2024-11-25T12:46:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741846_1022 (size=64) 2024-11-25T12:46:02,079 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:02,081 INFO [StoreOpener-23f87995af9a0ade941413c0db58171f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,083 INFO [StoreOpener-23f87995af9a0ade941413c0db58171f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23f87995af9a0ade941413c0db58171f columnFamilyName a 2024-11-25T12:46:02,083 DEBUG [StoreOpener-23f87995af9a0ade941413c0db58171f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:02,084 INFO [StoreOpener-23f87995af9a0ade941413c0db58171f-1 {}] regionserver.HStore(327): Store=23f87995af9a0ade941413c0db58171f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:02,084 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,086 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,086 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,087 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,087 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,089 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,093 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:02,094 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 23f87995af9a0ade941413c0db58171f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67993435, jitterRate=0.013181135058403015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:02,094 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 23f87995af9a0ade941413c0db58171f: Writing region info on filesystem at 1732538762080Initializing all the Stores at 1732538762081 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538762081Cleaning up temporary data from old regions at 1732538762087 (+6 ms)Region opened successfully at 1732538762094 (+7 ms) 2024-11-25T12:46:02,094 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 23f87995af9a0ade941413c0db58171f, disabling compactions & flushes 2024-11-25T12:46:02,094 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f. 2024-11-25T12:46:02,094 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f. 2024-11-25T12:46:02,094 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f. after waiting 0 ms 2024-11-25T12:46:02,095 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f. 2024-11-25T12:46:02,095 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f. 2024-11-25T12:46:02,095 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 23f87995af9a0ade941413c0db58171f: Waiting for close lock at 1732538762094Disabling compacts and flushes for region at 1732538762094Disabling writes for close at 1732538762094Writing region close event to WAL at 1732538762095 (+1 ms)Closed at 1732538762095 2024-11-25T12:46:02,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741845_1021 (size=95) 2024-11-25T12:46:02,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741845_1021 (size=95) 2024-11-25T12:46:02,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741845_1021 (size=95) 2024-11-25T12:46:02,105 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:02,105 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-06333675:(num 1732538762039) 2024-11-25T12:46:02,107 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T12:46:02,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741847_1023 (size=320) 2024-11-25T12:46:02,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741847_1023 (size=320) 2024-11-25T12:46:02,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741847_1023 (size=320) 2024-11-25T12:46:02,135 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T12:46:02,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741848_1024 (size=253) 2024-11-25T12:46:02,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741848_1024 (size=253) 2024-11-25T12:46:02,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741848_1024 (size=253) 2024-11-25T12:46:02,190 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2, size=253 (253bytes) 2024-11-25T12:46:02,191 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2 2024-11-25T12:46:02,192 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2 after 1ms 2024-11-25T12:46:02,198 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:02,198 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2 took 8ms 2024-11-25T12:46:02,204 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2 so closing down 2024-11-25T12:46:02,204 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:02,206 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-25T12:46:02,208 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002-wal-2.temp 2024-11-25T12:46:02,209 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:02,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741849_1025 (size=253) 2024-11-25T12:46:02,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741849_1025 (size=253) 2024-11-25T12:46:02,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741849_1025 (size=253) 2024-11-25T12:46:02,224 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:02,227 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002 2024-11-25T12:46:02,227 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-25T12:46:02,227 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2, journal: Splitting hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2, size=253 (253bytes) at 1732538762191Finishing writing output for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2 so closing down at 1732538762204 (+13 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002-wal-2.temp at 1732538762208 (+4 ms)3 split writer threads finished at 1732538762209 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732538762224 (+15 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002 at 1732538762227 (+3 ms)Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732538762227 2024-11-25T12:46:02,249 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1, size=320 (320bytes) 2024-11-25T12:46:02,249 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1 2024-11-25T12:46:02,250 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1 after 1ms 2024-11-25T12:46:02,255 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:02,256 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1 took 7ms 2024-11-25T12:46:02,260 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1 so closing down 2024-11-25T12:46:02,260 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:02,262 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-25T12:46:02,265 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000001-wal-1.temp 2024-11-25T12:46:02,265 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:02,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741850_1026 (size=320) 2024-11-25T12:46:02,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741850_1026 (size=320) 2024-11-25T12:46:02,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741850_1026 (size=320) 2024-11-25T12:46:02,283 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:02,288 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:02,292 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002, length=253 2024-11-25T12:46:02,295 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002 2024-11-25T12:46:02,295 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-25T12:46:02,295 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1, journal: Splitting hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1, size=320 (320bytes) at 1732538762249Finishing writing output for hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1 so closing down at 1732538762260 (+11 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000001-wal-1.temp at 1732538762265 (+5 ms)3 split writer threads finished at 1732538762265Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732538762283 (+18 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002 at 1732538762295 (+12 ms)Processed 2 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732538762295 2024-11-25T12:46:02,295 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:02,301 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:02,319 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal.1732538762302, exclude list is [], retry=0 2024-11-25T12:46:02,323 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:02,324 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:02,324 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:02,329 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal.1732538762302 2024-11-25T12:46:02,330 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:02,330 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 23f87995af9a0ade941413c0db58171f, NAME => 'testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:02,330 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:02,330 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,330 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,335 INFO [StoreOpener-23f87995af9a0ade941413c0db58171f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,338 INFO [StoreOpener-23f87995af9a0ade941413c0db58171f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23f87995af9a0ade941413c0db58171f columnFamilyName a 2024-11-25T12:46:02,338 DEBUG [StoreOpener-23f87995af9a0ade941413c0db58171f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:02,340 INFO [StoreOpener-23f87995af9a0ade941413c0db58171f-1 {}] regionserver.HStore(327): Store=23f87995af9a0ade941413c0db58171f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:02,340 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,345 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,350 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002 2024-11-25T12:46:02,355 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:02,358 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002 2024-11-25T12:46:02,358 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 23f87995af9a0ade941413c0db58171f 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-25T12:46:02,383 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/.tmp/a/46f19ce9c87d4959bbcc8113f74e4b32 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732538762106/Put/seqid=0 2024-11-25T12:46:02,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741852_1028 (size=5170) 2024-11-25T12:46:02,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741852_1028 (size=5170) 2024-11-25T12:46:02,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741852_1028 (size=5170) 2024-11-25T12:46:02,393 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/.tmp/a/46f19ce9c87d4959bbcc8113f74e4b32 2024-11-25T12:46:02,414 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/.tmp/a/46f19ce9c87d4959bbcc8113f74e4b32 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/a/46f19ce9c87d4959bbcc8113f74e4b32 2024-11-25T12:46:02,427 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/a/46f19ce9c87d4959bbcc8113f74e4b32, entries=2, sequenceid=2, filesize=5.0 K 2024-11-25T12:46:02,427 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 23f87995af9a0ade941413c0db58171f in 69ms, sequenceid=2, compaction requested=false; wal=null 2024-11-25T12:46:02,428 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/0000000000000000002 2024-11-25T12:46:02,429 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,429 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,432 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 23f87995af9a0ade941413c0db58171f 2024-11-25T12:46:02,437 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/23f87995af9a0ade941413c0db58171f/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-25T12:46:02,439 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 23f87995af9a0ade941413c0db58171f; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65248713, jitterRate=-0.02771840989589691}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:02,439 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 23f87995af9a0ade941413c0db58171f: Writing region info on filesystem at 1732538762331Initializing all the Stores at 1732538762333 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538762333Obtaining lock to block concurrent updates at 1732538762358 (+25 ms)Preparing flush snapshotting stores in 23f87995af9a0ade941413c0db58171f at 1732538762358Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732538762359 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732538762031.23f87995af9a0ade941413c0db58171f. at 1732538762359Flushing 23f87995af9a0ade941413c0db58171f/a: creating writer at 1732538762359Flushing 23f87995af9a0ade941413c0db58171f/a: appending metadata at 1732538762381 (+22 ms)Flushing 23f87995af9a0ade941413c0db58171f/a: closing flushed file at 1732538762382 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77066865: reopening flushed file at 1732538762409 (+27 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 23f87995af9a0ade941413c0db58171f in 69ms, sequenceid=2, compaction requested=false; wal=null at 1732538762427 (+18 ms)Cleaning up temporary data from old regions at 1732538762429 (+2 ms)Region opened successfully at 1732538762439 (+10 ms) 2024-11-25T12:46:02,475 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=378 (was 368) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:39968 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:52976 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:53022 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:39858 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:41802 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:41906 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=767 (was 683) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=620 (was 620), ProcessCount=11 (was 11), AvailableMemoryMB=2589 (was 2624) 2024-11-25T12:46:02,491 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=378, OpenFileDescriptor=767, MaxFileDescriptor=1048576, SystemLoadAverage=620, ProcessCount=11, AvailableMemoryMB=2588 2024-11-25T12:46:02,514 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:02,518 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:02,520 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:02,523 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-49085054, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-49085054, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:02,540 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-49085054/hregion-49085054.1732538762524, exclude list is [], retry=0 2024-11-25T12:46:02,544 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:02,544 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:02,546 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:02,552 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-49085054/hregion-49085054.1732538762524 2024-11-25T12:46:02,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:02,555 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 130646d73437b3ea868a6bc3c7a22395, NAME => 'testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:02,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741854_1030 (size=64) 2024-11-25T12:46:02,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741854_1030 (size=64) 2024-11-25T12:46:02,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741854_1030 (size=64) 2024-11-25T12:46:02,576 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:02,581 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,584 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130646d73437b3ea868a6bc3c7a22395 columnFamilyName a 2024-11-25T12:46:02,584 DEBUG [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:02,585 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(327): Store=130646d73437b3ea868a6bc3c7a22395/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:02,586 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,588 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130646d73437b3ea868a6bc3c7a22395 columnFamilyName b 2024-11-25T12:46:02,588 DEBUG [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:02,589 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(327): Store=130646d73437b3ea868a6bc3c7a22395/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:02,589 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,594 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130646d73437b3ea868a6bc3c7a22395 columnFamilyName c 2024-11-25T12:46:02,594 DEBUG [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:02,595 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(327): Store=130646d73437b3ea868a6bc3c7a22395/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:02,596 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,598 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,600 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,600 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,601 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:02,603 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:02,612 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:02,613 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 130646d73437b3ea868a6bc3c7a22395; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75325354, jitterRate=0.12243524193763733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:02,614 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 130646d73437b3ea868a6bc3c7a22395: Writing region info on filesystem at 1732538762576Initializing all the Stores at 1732538762578 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538762578Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538762581 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538762581Cleaning up temporary data from old regions at 1732538762600 (+19 ms)Region opened successfully at 1732538762613 (+13 ms) 2024-11-25T12:46:02,614 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 130646d73437b3ea868a6bc3c7a22395, disabling compactions & flushes 2024-11-25T12:46:02,614 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:02,614 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:02,614 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. after waiting 0 ms 2024-11-25T12:46:02,614 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:02,615 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:02,615 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 130646d73437b3ea868a6bc3c7a22395: Waiting for close lock at 1732538762614Disabling compacts and flushes for region at 1732538762614Disabling writes for close at 1732538762614Writing region close event to WAL at 1732538762615 (+1 ms)Closed at 1732538762615 2024-11-25T12:46:02,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741853_1029 (size=95) 2024-11-25T12:46:02,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741853_1029 (size=95) 2024-11-25T12:46:02,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741853_1029 (size=95) 2024-11-25T12:46:02,626 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:02,626 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-49085054:(num 1732538762524) 2024-11-25T12:46:02,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:02,629 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:02,643 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630, exclude list is [], retry=0 2024-11-25T12:46:02,647 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:02,647 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:02,647 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:02,652 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 2024-11-25T12:46:02,652 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:02,967 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630, size=0 (0bytes) 2024-11-25T12:46:02,967 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 might be still open, length is 0 2024-11-25T12:46:02,968 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 2024-11-25T12:46:02,969 WARN [IPC Server handler 0 on default port 35793 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-25T12:46:02,969 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 after 1ms 2024-11-25T12:46:03,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741827_1003 (size=196) 2024-11-25T12:46:03,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741829_1005 (size=34) 2024-11-25T12:46:03,293 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:40012 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33377:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40012 dst: /127.0.0.1:33377 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33377 remote=/127.0.0.1:40012]. Total timeout mills is 60000, 59622 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:03,294 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:53060 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53060 dst: /127.0.0.1:34959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:03,295 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:41948 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41948 dst: /127.0.0.1:35653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:03,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741855_1032 (size=263633) 2024-11-25T12:46:03,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741855_1032 (size=263633) 2024-11-25T12:46:03,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741855_1032 (size=263633) 2024-11-25T12:46:06,041 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:46:06,115 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T12:46:06,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T12:46:06,127 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:06,129 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T12:46:06,129 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T12:46:06,129 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T12:46:06,130 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:06,131 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-25T12:46:06,131 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:06,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741832_1008 (size=32) 2024-11-25T12:46:06,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741828_1004 (size=1189) 2024-11-25T12:46:06,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741831_1007 (size=1321) 2024-11-25T12:46:06,970 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 after 4002ms 2024-11-25T12:46:06,979 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:06,981 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 took 4013ms 2024-11-25T12:46:06,985 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732538762630.temp 2024-11-25T12:46:06,987 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000000001-wal.1732538762630.temp 2024-11-25T12:46:07,117 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630; continuing. 2024-11-25T12:46:07,117 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 so closing down 2024-11-25T12:46:07,117 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:07,117 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:07,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741856_1033 (size=263641) 2024-11-25T12:46:07,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741856_1033 (size=263641) 2024-11-25T12:46:07,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741856_1033 (size=263641) 2024-11-25T12:46:07,122 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000000001-wal.1732538762630.temp (wrote 3002 edits, skipped 0 edits in 70 ms) 2024-11-25T12:46:07,124 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000000001-wal.1732538762630.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000003002 2024-11-25T12:46:07,125 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 143 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T12:46:07,125 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630, journal: Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630, size=0 (0bytes) at 1732538762967Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000000001-wal.1732538762630.temp at 1732538766987 (+4020 ms)Split 1024 edits, skipped 0 edits. at 1732538767052 (+65 ms)Split 2048 edits, skipped 0 edits. at 1732538767087 (+35 ms)Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 so closing down at 1732538767117 (+30 ms)3 split writer threads finished at 1732538767118 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000000001-wal.1732538762630.temp (wrote 3002 edits, skipped 0 edits in 70 ms) at 1732538767122 (+4 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000000001-wal.1732538762630.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000003002 at 1732538767124 (+2 ms)Processed 3002 edits across 1 Regions in 143 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630, size=0, length=0, corrupted=false, cancelled=false at 1732538767125 (+1 ms) 2024-11-25T12:46:07,127 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538762630 2024-11-25T12:46:07,128 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000003002 2024-11-25T12:46:07,129 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:07,131 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:07,144 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538767132, exclude list is [], retry=0 2024-11-25T12:46:07,147 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:07,148 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:07,148 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:07,151 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538767132 2024-11-25T12:46:07,151 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:07,151 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:07,153 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:07,155 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130646d73437b3ea868a6bc3c7a22395 columnFamilyName a 2024-11-25T12:46:07,155 DEBUG [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:07,156 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(327): Store=130646d73437b3ea868a6bc3c7a22395/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:07,156 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:07,157 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130646d73437b3ea868a6bc3c7a22395 columnFamilyName b 2024-11-25T12:46:07,157 DEBUG [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:07,158 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(327): Store=130646d73437b3ea868a6bc3c7a22395/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:07,158 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:07,159 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130646d73437b3ea868a6bc3c7a22395 columnFamilyName c 2024-11-25T12:46:07,159 DEBUG [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:07,160 INFO [StoreOpener-130646d73437b3ea868a6bc3c7a22395-1 {}] regionserver.HStore(327): Store=130646d73437b3ea868a6bc3c7a22395/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:07,160 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:07,161 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:07,163 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:07,164 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000003002 2024-11-25T12:46:07,167 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:07,220 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T12:46:07,622 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 130646d73437b3ea868a6bc3c7a22395 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T12:46:07,673 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/a/0776ba67630e45dbaa1783b136a6993f is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732538762662/Put/seqid=0 2024-11-25T12:46:07,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741858_1035 (size=50463) 2024-11-25T12:46:07,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741858_1035 (size=50463) 2024-11-25T12:46:07,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741858_1035 (size=50463) 2024-11-25T12:46:07,683 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/a/0776ba67630e45dbaa1783b136a6993f 2024-11-25T12:46:07,692 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/a/0776ba67630e45dbaa1783b136a6993f as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/a/0776ba67630e45dbaa1783b136a6993f 2024-11-25T12:46:07,706 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/a/0776ba67630e45dbaa1783b136a6993f, entries=754, sequenceid=754, filesize=49.3 K 2024-11-25T12:46:07,707 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 130646d73437b3ea868a6bc3c7a22395 in 85ms, sequenceid=754, compaction requested=false; wal=null 2024-11-25T12:46:07,735 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T12:46:07,735 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 130646d73437b3ea868a6bc3c7a22395 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T12:46:07,747 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/a/3b8c21c58aed4c24af45ac80e336f034 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732538762720/Put/seqid=0 2024-11-25T12:46:07,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741859_1036 (size=20072) 2024-11-25T12:46:07,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741859_1036 (size=20072) 2024-11-25T12:46:07,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741859_1036 (size=20072) 2024-11-25T12:46:07,767 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/a/3b8c21c58aed4c24af45ac80e336f034 2024-11-25T12:46:07,797 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/b/99de46c440d9476aaf557c67990030fa is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732538762770/Put/seqid=0 2024-11-25T12:46:07,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741860_1037 (size=35835) 2024-11-25T12:46:07,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741860_1037 (size=35835) 2024-11-25T12:46:07,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741860_1037 (size=35835) 2024-11-25T12:46:07,811 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/b/99de46c440d9476aaf557c67990030fa 2024-11-25T12:46:07,819 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/a/3b8c21c58aed4c24af45ac80e336f034 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/a/3b8c21c58aed4c24af45ac80e336f034 2024-11-25T12:46:07,828 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/a/3b8c21c58aed4c24af45ac80e336f034, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-25T12:46:07,830 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/b/99de46c440d9476aaf557c67990030fa as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/b/99de46c440d9476aaf557c67990030fa 2024-11-25T12:46:07,839 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/b/99de46c440d9476aaf557c67990030fa, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-25T12:46:07,839 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 130646d73437b3ea868a6bc3c7a22395 in 104ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-25T12:46:07,859 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T12:46:07,860 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 130646d73437b3ea868a6bc3c7a22395 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T12:46:07,869 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/b/558177d9a87c45bdad07782794c147f1 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732538762796/Put/seqid=0 2024-11-25T12:46:07,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741861_1038 (size=35082) 2024-11-25T12:46:07,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741861_1038 (size=35082) 2024-11-25T12:46:07,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741861_1038 (size=35082) 2024-11-25T12:46:07,881 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/b/558177d9a87c45bdad07782794c147f1 2024-11-25T12:46:07,909 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/c/524b42317d0948589e8da41820374174 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732538762842/Put/seqid=0 2024-11-25T12:46:07,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741862_1039 (size=20825) 2024-11-25T12:46:07,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741862_1039 (size=20825) 2024-11-25T12:46:07,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741862_1039 (size=20825) 2024-11-25T12:46:07,921 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/c/524b42317d0948589e8da41820374174 2024-11-25T12:46:07,941 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/b/558177d9a87c45bdad07782794c147f1 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/b/558177d9a87c45bdad07782794c147f1 2024-11-25T12:46:07,950 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/b/558177d9a87c45bdad07782794c147f1, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-25T12:46:07,952 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/c/524b42317d0948589e8da41820374174 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/c/524b42317d0948589e8da41820374174 2024-11-25T12:46:07,961 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/c/524b42317d0948589e8da41820374174, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-25T12:46:07,961 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 130646d73437b3ea868a6bc3c7a22395 in 101ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-25T12:46:07,974 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732538762911/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:07,977 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000003002 2024-11-25T12:46:07,977 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T12:46:07,978 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 130646d73437b3ea868a6bc3c7a22395 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-25T12:46:07,987 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/c/16f825a411744b01b9bc501c8989e6fc is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732538762854/Put/seqid=0 2024-11-25T12:46:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741863_1040 (size=50301) 2024-11-25T12:46:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741863_1040 (size=50301) 2024-11-25T12:46:08,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741863_1040 (size=50301) 2024-11-25T12:46:08,006 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/c/16f825a411744b01b9bc501c8989e6fc 2024-11-25T12:46:08,014 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 16f825a411744b01b9bc501c8989e6fc 2024-11-25T12:46:08,016 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/.tmp/c/16f825a411744b01b9bc501c8989e6fc as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/c/16f825a411744b01b9bc501c8989e6fc 2024-11-25T12:46:08,022 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 16f825a411744b01b9bc501c8989e6fc 2024-11-25T12:46:08,023 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/c/16f825a411744b01b9bc501c8989e6fc, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-25T12:46:08,023 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 130646d73437b3ea868a6bc3c7a22395 in 45ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-25T12:46:08,024 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/0000000000000003002 2024-11-25T12:46:08,025 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:08,025 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:08,026 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:46:08,028 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 130646d73437b3ea868a6bc3c7a22395 2024-11-25T12:46:08,031 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenIntoWAL/130646d73437b3ea868a6bc3c7a22395/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-25T12:46:08,033 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 130646d73437b3ea868a6bc3c7a22395; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62579994, jitterRate=-0.06748542189598083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:46:08,033 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 130646d73437b3ea868a6bc3c7a22395: Writing region info on filesystem at 1732538767151Initializing all the Stores at 1732538767153 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538767153Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538767153Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538767153Cleaning up temporary data from old regions at 1732538768025 (+872 ms)Region opened successfully at 1732538768033 (+8 ms) 2024-11-25T12:46:08,108 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 130646d73437b3ea868a6bc3c7a22395, disabling compactions & flushes 2024-11-25T12:46:08,109 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:08,109 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:08,109 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. after waiting 0 ms 2024-11-25T12:46:08,109 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:08,116 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732538762515.130646d73437b3ea868a6bc3c7a22395. 2024-11-25T12:46:08,116 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 130646d73437b3ea868a6bc3c7a22395: Waiting for close lock at 1732538768108Disabling compacts and flushes for region at 1732538768108Disabling writes for close at 1732538768109 (+1 ms)Writing region close event to WAL at 1732538768115 (+6 ms)Closed at 1732538768115 2024-11-25T12:46:08,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741857_1034 (size=95) 2024-11-25T12:46:08,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741857_1034 (size=95) 2024-11-25T12:46:08,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741857_1034 (size=95) 2024-11-25T12:46:08,125 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538767132 not finished, retry = 0 2024-11-25T12:46:08,233 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:08,233 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538767132) 2024-11-25T12:46:08,250 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=396 (was 378) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-455498575_22 at /127.0.0.1:56190 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@3eecba9b[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40517 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-455498575_22 at /127.0.0.1:50034 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@2d8e3dcf[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@5144c8be[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44747 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35793 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:40517 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-455498575_22 at /127.0.0.1:56520 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-455498575_22 at /127.0.0.1:56498 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:35793 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:44747 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@349912fb[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=847 (was 767) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=639 (was 620) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2044 (was 2588) 2024-11-25T12:46:08,267 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=396, OpenFileDescriptor=847, MaxFileDescriptor=1048576, SystemLoadAverage=639, ProcessCount=11, AvailableMemoryMB=2042 2024-11-25T12:46:08,293 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:08,299 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:08,300 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:08,303 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-56539936, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-56539936, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:08,317 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-56539936/hregion-56539936.1732538768303, exclude list is [], retry=0 2024-11-25T12:46:08,321 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:08,321 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:08,322 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:08,328 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-56539936/hregion-56539936.1732538768303 2024-11-25T12:46:08,328 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:08,329 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0d5ce4c0b00461db636d685848b30331, NAME => 'test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:08,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741865_1042 (size=43) 2024-11-25T12:46:08,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741865_1042 (size=43) 2024-11-25T12:46:08,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741865_1042 (size=43) 2024-11-25T12:46:08,352 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:08,354 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,356 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d5ce4c0b00461db636d685848b30331 columnFamilyName a 2024-11-25T12:46:08,357 DEBUG [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:08,357 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(327): Store=0d5ce4c0b00461db636d685848b30331/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:08,358 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,360 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d5ce4c0b00461db636d685848b30331 columnFamilyName b 2024-11-25T12:46:08,360 DEBUG [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:08,361 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(327): Store=0d5ce4c0b00461db636d685848b30331/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:08,361 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,363 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d5ce4c0b00461db636d685848b30331 columnFamilyName c 2024-11-25T12:46:08,363 DEBUG [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:08,364 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(327): Store=0d5ce4c0b00461db636d685848b30331/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:08,365 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,365 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,366 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,367 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,367 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,367 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:08,369 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:08,371 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:08,372 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d5ce4c0b00461db636d685848b30331; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72553342, jitterRate=0.0811290442943573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:08,373 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d5ce4c0b00461db636d685848b30331: Writing region info on filesystem at 1732538768353Initializing all the Stores at 1732538768354 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538768354Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538768354Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538768354Cleaning up temporary data from old regions at 1732538768367 (+13 ms)Region opened successfully at 1732538768373 (+6 ms) 2024-11-25T12:46:08,373 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d5ce4c0b00461db636d685848b30331, disabling compactions & flushes 2024-11-25T12:46:08,373 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. 2024-11-25T12:46:08,373 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. 2024-11-25T12:46:08,373 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. after waiting 0 ms 2024-11-25T12:46:08,373 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. 2024-11-25T12:46:08,373 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. 2024-11-25T12:46:08,373 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d5ce4c0b00461db636d685848b30331: Waiting for close lock at 1732538768373Disabling compacts and flushes for region at 1732538768373Disabling writes for close at 1732538768373Writing region close event to WAL at 1732538768373Closed at 1732538768373 2024-11-25T12:46:08,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741864_1041 (size=95) 2024-11-25T12:46:08,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741864_1041 (size=95) 2024-11-25T12:46:08,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741864_1041 (size=95) 2024-11-25T12:46:08,380 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:08,380 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-56539936:(num 1732538768303) 2024-11-25T12:46:08,381 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:08,383 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:08,400 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384, exclude list is [], retry=0 2024-11-25T12:46:08,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:08,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:08,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:08,406 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384 2024-11-25T12:46:08,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:08,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741866_1043 (size=263359) 2024-11-25T12:46:08,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741866_1043 (size=263359) 2024-11-25T12:46:08,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741866_1043 (size=263359) 2024-11-25T12:46:08,654 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384, size=257.2 K (263359bytes) 2024-11-25T12:46:08,654 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384 2024-11-25T12:46:08,655 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384 after 1ms 2024-11-25T12:46:08,659 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:08,660 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384 took 6ms 2024-11-25T12:46:08,669 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732538768384.temp 2024-11-25T12:46:08,671 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000000001-wal.1732538768384.temp 2024-11-25T12:46:08,733 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384 so closing down 2024-11-25T12:46:08,733 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:08,734 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:08,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741867_1044 (size=263359) 2024-11-25T12:46:08,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741867_1044 (size=263359) 2024-11-25T12:46:08,743 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000000001-wal.1732538768384.temp (wrote 3000 edits, skipped 0 edits in 44 ms) 2024-11-25T12:46:08,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741867_1044 (size=263359) 2024-11-25T12:46:08,747 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000000001-wal.1732538768384.temp to hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003000 2024-11-25T12:46:08,748 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-25T12:46:08,748 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384, journal: Splitting hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384, size=257.2 K (263359bytes) at 1732538768654Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000000001-wal.1732538768384.temp at 1732538768671 (+17 ms)Split 1024 edits, skipped 0 edits. at 1732538768687 (+16 ms)Split 2048 edits, skipped 0 edits. at 1732538768709 (+22 ms)Finishing writing output for hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384 so closing down at 1732538768733 (+24 ms)3 split writer threads finished at 1732538768734 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000000001-wal.1732538768384.temp (wrote 3000 edits, skipped 0 edits in 44 ms) at 1732538768744 (+10 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000000001-wal.1732538768384.temp to hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003000 at 1732538768747 (+3 ms)Processed 3000 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1732538768748 (+1 ms) 2024-11-25T12:46:08,759 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768384 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538768384 2024-11-25T12:46:08,763 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003000 2024-11-25T12:46:08,763 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:08,770 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:08,792 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772, exclude list is [], retry=0 2024-11-25T12:46:08,798 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:08,799 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:08,799 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:08,812 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772 2024-11-25T12:46:08,812 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:08,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741868_1045 (size=263486) 2024-11-25T12:46:08,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741868_1045 (size=263486) 2024-11-25T12:46:08,997 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772 not finished, retry = 0 2024-11-25T12:46:08,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741868_1045 (size=263486) 2024-11-25T12:46:09,113 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772, size=257.3 K (263486bytes) 2024-11-25T12:46:09,113 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772 2024-11-25T12:46:09,114 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772 after 1ms 2024-11-25T12:46:09,116 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:09,118 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772 took 5ms 2024-11-25T12:46:09,122 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732538768772.temp 2024-11-25T12:46:09,123 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003001-wal.1732538768772.temp 2024-11-25T12:46:09,177 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772 so closing down 2024-11-25T12:46:09,177 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:09,179 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:09,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741869_1046 (size=263486) 2024-11-25T12:46:09,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741869_1046 (size=263486) 2024-11-25T12:46:09,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741869_1046 (size=263486) 2024-11-25T12:46:09,187 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003001-wal.1732538768772.temp (wrote 3000 edits, skipped 0 edits in 38 ms) 2024-11-25T12:46:09,189 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003001-wal.1732538768772.temp to hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000006000 2024-11-25T12:46:09,189 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-25T12:46:09,189 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772, journal: Splitting hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772, size=257.3 K (263486bytes) at 1732538769113Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003001-wal.1732538768772.temp at 1732538769124 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732538769138 (+14 ms)Split 2048 edits, skipped 0 edits. at 1732538769158 (+20 ms)Finishing writing output for hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772 so closing down at 1732538769177 (+19 ms)3 split writer threads finished at 1732538769179 (+2 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003001-wal.1732538768772.temp (wrote 3000 edits, skipped 0 edits in 38 ms) at 1732538769187 (+8 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003001-wal.1732538768772.temp to hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000006000 at 1732538769189 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1732538769189 2024-11-25T12:46:09,193 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538768772 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538768772 2024-11-25T12:46:09,194 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000006000 2024-11-25T12:46:09,195 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:09,197 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/test2727-manual,16010,1732538768287, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:09,218 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538769198, exclude list is [], retry=0 2024-11-25T12:46:09,221 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:09,222 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:09,222 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:09,228 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732538768287/wal.1732538769198 2024-11-25T12:46:09,228 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:09,228 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d5ce4c0b00461db636d685848b30331, NAME => 'test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:09,229 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:09,229 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,229 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,234 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,236 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d5ce4c0b00461db636d685848b30331 columnFamilyName a 2024-11-25T12:46:09,236 DEBUG [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:09,237 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(327): Store=0d5ce4c0b00461db636d685848b30331/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:09,237 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,238 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d5ce4c0b00461db636d685848b30331 columnFamilyName b 2024-11-25T12:46:09,238 DEBUG [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:09,239 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(327): Store=0d5ce4c0b00461db636d685848b30331/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:09,240 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,242 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d5ce4c0b00461db636d685848b30331 columnFamilyName c 2024-11-25T12:46:09,242 DEBUG [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:09,244 INFO [StoreOpener-0d5ce4c0b00461db636d685848b30331-1 {}] regionserver.HStore(327): Store=0d5ce4c0b00461db636d685848b30331/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:09,244 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,248 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,249 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003000 2024-11-25T12:46:09,253 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:09,337 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003000 2024-11-25T12:46:09,338 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000006000 2024-11-25T12:46:09,342 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:09,402 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000006000 2024-11-25T12:46:09,402 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0d5ce4c0b00461db636d685848b30331 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-25T12:46:09,437 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/a/c3584ca71f944c5d9584f6770719cdfb is 41, key is test2727/a:100/1732538768820/Put/seqid=0 2024-11-25T12:46:09,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741871_1048 (size=84227) 2024-11-25T12:46:09,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741871_1048 (size=84227) 2024-11-25T12:46:09,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741871_1048 (size=84227) 2024-11-25T12:46:09,445 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/a/c3584ca71f944c5d9584f6770719cdfb 2024-11-25T12:46:09,490 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/b/03fbe8ded1334d30a3d31ae101a3be6a is 41, key is test2727/b:100/1732538768882/Put/seqid=0 2024-11-25T12:46:09,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741872_1049 (size=84609) 2024-11-25T12:46:09,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741872_1049 (size=84609) 2024-11-25T12:46:09,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741872_1049 (size=84609) 2024-11-25T12:46:09,504 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/b/03fbe8ded1334d30a3d31ae101a3be6a 2024-11-25T12:46:09,549 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/c/533cb92e077840578900b24d9216552e is 41, key is test2727/c:100/1732538768943/Put/seqid=0 2024-11-25T12:46:09,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741873_1050 (size=84609) 2024-11-25T12:46:09,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741873_1050 (size=84609) 2024-11-25T12:46:09,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741873_1050 (size=84609) 2024-11-25T12:46:09,563 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/c/533cb92e077840578900b24d9216552e 2024-11-25T12:46:09,572 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/a/c3584ca71f944c5d9584f6770719cdfb as hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/a/c3584ca71f944c5d9584f6770719cdfb 2024-11-25T12:46:09,580 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/a/c3584ca71f944c5d9584f6770719cdfb, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-25T12:46:09,582 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/b/03fbe8ded1334d30a3d31ae101a3be6a as hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/b/03fbe8ded1334d30a3d31ae101a3be6a 2024-11-25T12:46:09,589 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/b/03fbe8ded1334d30a3d31ae101a3be6a, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-25T12:46:09,590 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/.tmp/c/533cb92e077840578900b24d9216552e as hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/c/533cb92e077840578900b24d9216552e 2024-11-25T12:46:09,598 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/c/533cb92e077840578900b24d9216552e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-25T12:46:09,598 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0d5ce4c0b00461db636d685848b30331 in 196ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-25T12:46:09,599 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000003000 2024-11-25T12:46:09,599 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/0000000000000006000 2024-11-25T12:46:09,601 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,601 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,603 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:09,605 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d5ce4c0b00461db636d685848b30331 2024-11-25T12:46:09,611 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/test2727/0d5ce4c0b00461db636d685848b30331/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-25T12:46:09,612 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d5ce4c0b00461db636d685848b30331; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66826101, jitterRate=-0.004213497042655945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:09,613 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d5ce4c0b00461db636d685848b30331: Writing region info on filesystem at 1732538769229Initializing all the Stores at 1732538769231 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538769231Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538769234 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538769234Obtaining lock to block concurrent updates at 1732538769402 (+168 ms)Preparing flush snapshotting stores in 0d5ce4c0b00461db636d685848b30331 at 1732538769402Finished memstore snapshotting test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732538769402Flushing stores of test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. at 1732538769402Flushing 0d5ce4c0b00461db636d685848b30331/a: creating writer at 1732538769402Flushing 0d5ce4c0b00461db636d685848b30331/a: appending metadata at 1732538769436 (+34 ms)Flushing 0d5ce4c0b00461db636d685848b30331/a: closing flushed file at 1732538769436Flushing 0d5ce4c0b00461db636d685848b30331/b: creating writer at 1732538769457 (+21 ms)Flushing 0d5ce4c0b00461db636d685848b30331/b: appending metadata at 1732538769488 (+31 ms)Flushing 0d5ce4c0b00461db636d685848b30331/b: closing flushed file at 1732538769488Flushing 0d5ce4c0b00461db636d685848b30331/c: creating writer at 1732538769514 (+26 ms)Flushing 0d5ce4c0b00461db636d685848b30331/c: appending metadata at 1732538769548 (+34 ms)Flushing 0d5ce4c0b00461db636d685848b30331/c: closing flushed file at 1732538769548Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c51d828: reopening flushed file at 1732538769571 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b4df6bd: reopening flushed file at 1732538769581 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c00a7: reopening flushed file at 1732538769589 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0d5ce4c0b00461db636d685848b30331 in 196ms, sequenceid=6000, compaction requested=false; wal=null at 1732538769598 (+9 ms)Cleaning up temporary data from old regions at 1732538769601 (+3 ms)Region opened successfully at 1732538769613 (+12 ms) 2024-11-25T12:46:09,614 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-25T12:46:09,615 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d5ce4c0b00461db636d685848b30331, disabling compactions & flushes 2024-11-25T12:46:09,615 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. 2024-11-25T12:46:09,615 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. 2024-11-25T12:46:09,615 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. after waiting 0 ms 2024-11-25T12:46:09,615 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. 2024-11-25T12:46:09,616 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732538768294.0d5ce4c0b00461db636d685848b30331. 2024-11-25T12:46:09,616 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d5ce4c0b00461db636d685848b30331: Waiting for close lock at 1732538769614Disabling compacts and flushes for region at 1732538769614Disabling writes for close at 1732538769615 (+1 ms)Writing region close event to WAL at 1732538769616 (+1 ms)Closed at 1732538769616 2024-11-25T12:46:09,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741870_1047 (size=95) 2024-11-25T12:46:09,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741870_1047 (size=95) 2024-11-25T12:46:09,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741870_1047 (size=95) 2024-11-25T12:46:09,625 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:09,625 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538769198) 2024-11-25T12:46:09,639 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=400 (was 396) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:50034 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:56342 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:56520 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=909 (was 847) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=639 (was 639), ProcessCount=11 (was 11), AvailableMemoryMB=1797 (was 2042) 2024-11-25T12:46:09,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=400, OpenFileDescriptor=909, MaxFileDescriptor=1048576, SystemLoadAverage=639, ProcessCount=11, AvailableMemoryMB=1796 2024-11-25T12:46:09,666 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:09,673 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:09,674 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732538769674 2024-11-25T12:46:09,682 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674 2024-11-25T12:46:09,685 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:09,686 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b456c722ab228e8a946d16c137a7aedc, NAME => 'testSequentialEditLogSeqNum,,1732538769667.b456c722ab228e8a946d16c137a7aedc.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:09,686 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732538769667.b456c722ab228e8a946d16c137a7aedc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:09,687 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,687 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,687 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc doesn't exist for region: b456c722ab228e8a946d16c137a7aedc on table testSequentialEditLogSeqNum 2024-11-25T12:46:09,688 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: b456c722ab228e8a946d16c137a7aedc on table testSequentialEditLogSeqNum 2024-11-25T12:46:09,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741875_1052 (size=62) 2024-11-25T12:46:09,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741875_1052 (size=62) 2024-11-25T12:46:09,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741875_1052 (size=62) 2024-11-25T12:46:09,701 INFO [StoreOpener-b456c722ab228e8a946d16c137a7aedc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,703 INFO [StoreOpener-b456c722ab228e8a946d16c137a7aedc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b456c722ab228e8a946d16c137a7aedc columnFamilyName a 2024-11-25T12:46:09,703 DEBUG [StoreOpener-b456c722ab228e8a946d16c137a7aedc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:09,704 INFO [StoreOpener-b456c722ab228e8a946d16c137a7aedc-1 {}] regionserver.HStore(327): Store=b456c722ab228e8a946d16c137a7aedc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:09,704 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,704 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,705 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,705 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,705 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,707 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b456c722ab228e8a946d16c137a7aedc 2024-11-25T12:46:09,710 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:09,710 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b456c722ab228e8a946d16c137a7aedc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65450013, jitterRate=-0.02471880614757538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:09,711 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b456c722ab228e8a946d16c137a7aedc: Writing region info on filesystem at 1732538769687Initializing all the Stores at 1732538769701 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538769701Cleaning up temporary data from old regions at 1732538769705 (+4 ms)Region opened successfully at 1732538769711 (+6 ms) 2024-11-25T12:46:09,724 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b456c722ab228e8a946d16c137a7aedc 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-25T12:46:09,749 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/.tmp/a/9ff2fec90a79400f9cf74196ccaedf6b is 81, key is testSequentialEditLogSeqNum/a:x0/1732538769711/Put/seqid=0 2024-11-25T12:46:09,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741876_1053 (size=5833) 2024-11-25T12:46:09,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741876_1053 (size=5833) 2024-11-25T12:46:09,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741876_1053 (size=5833) 2024-11-25T12:46:09,757 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/.tmp/a/9ff2fec90a79400f9cf74196ccaedf6b 2024-11-25T12:46:09,764 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/.tmp/a/9ff2fec90a79400f9cf74196ccaedf6b as hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/a/9ff2fec90a79400f9cf74196ccaedf6b 2024-11-25T12:46:09,772 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/a/9ff2fec90a79400f9cf74196ccaedf6b, entries=10, sequenceid=13, filesize=5.7 K 2024-11-25T12:46:09,774 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b456c722ab228e8a946d16c137a7aedc in 50ms, sequenceid=13, compaction requested=false 2024-11-25T12:46:09,774 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b456c722ab228e8a946d16c137a7aedc: 2024-11-25T12:46:09,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:46:09,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:46:09,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:46:09,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:46:09,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:46:09,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741874_1051 (size=1845) 2024-11-25T12:46:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741874_1051 (size=1845) 2024-11-25T12:46:09,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741874_1051 (size=1845) 2024-11-25T12:46:09,808 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674, size=1.8 K (1845bytes) 2024-11-25T12:46:09,808 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674 2024-11-25T12:46:09,809 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674 after 1ms 2024-11-25T12:46:09,812 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:09,812 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674 took 4ms 2024-11-25T12:46:09,815 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674 so closing down 2024-11-25T12:46:09,815 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:09,816 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538769674.temp 2024-11-25T12:46:09,818 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/0000000000000000003-wal.1732538769674.temp 2024-11-25T12:46:09,819 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:09,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741877_1054 (size=1477) 2024-11-25T12:46:09,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741877_1054 (size=1477) 2024-11-25T12:46:09,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741877_1054 (size=1477) 2024-11-25T12:46:09,827 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/0000000000000000003-wal.1732538769674.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:09,829 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/0000000000000000003-wal.1732538769674.temp to hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/0000000000000000020 2024-11-25T12:46:09,829 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674, size=1.8 K, length=1845, corrupted=false, cancelled=false 2024-11-25T12:46:09,829 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674, journal: Splitting hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674, size=1.8 K (1845bytes) at 1732538769808Finishing writing output for hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674 so closing down at 1732538769815 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/0000000000000000003-wal.1732538769674.temp at 1732538769818 (+3 ms)3 split writer threads finished at 1732538769819 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/0000000000000000003-wal.1732538769674.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732538769827 (+8 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/0000000000000000003-wal.1732538769674.temp to hdfs://localhost:35793/hbase/data/default/testSequentialEditLogSeqNum/b456c722ab228e8a946d16c137a7aedc/recovered.edits/0000000000000000020 at 1732538769829 (+2 ms)Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:35793/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538769666/wal.1732538769674, size=1.8 K, length=1845, corrupted=false, cancelled=false at 1732538769829 2024-11-25T12:46:09,842 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=405 (was 400) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:50034 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:56342 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:56520 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=945 (was 909) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=639 (was 639), ProcessCount=11 (was 11), AvailableMemoryMB=1777 (was 1796) 2024-11-25T12:46:09,854 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=405, OpenFileDescriptor=945, MaxFileDescriptor=1048576, SystemLoadAverage=639, ProcessCount=11, AvailableMemoryMB=1772 2024-11-25T12:46:09,892 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:09,899 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:09,900 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:09,903 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-46948328, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-46948328, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:09,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-46948328/hregion-46948328.1732538769903, exclude list is [], retry=0 2024-11-25T12:46:09,920 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:09,920 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:09,920 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:09,957 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-46948328/hregion-46948328.1732538769903 2024-11-25T12:46:09,957 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:09,957 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4e5a1bce44d223654427771878a045ee, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:09,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741879_1056 (size=70) 2024-11-25T12:46:09,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741879_1056 (size=70) 2024-11-25T12:46:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741879_1056 (size=70) 2024-11-25T12:46:09,968 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:09,969 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,970 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName a 2024-11-25T12:46:09,971 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:09,971 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:09,971 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,972 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName b 2024-11-25T12:46:09,972 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:09,973 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:09,973 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,974 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName c 2024-11-25T12:46:09,974 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:09,975 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:09,975 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,976 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,977 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,978 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,978 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,978 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:09,979 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:09,982 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:09,982 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e5a1bce44d223654427771878a045ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73165296, jitterRate=0.09024786949157715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:09,983 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e5a1bce44d223654427771878a045ee: Writing region info on filesystem at 1732538769968Initializing all the Stores at 1732538769969 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538769969Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538769969Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538769969Cleaning up temporary data from old regions at 1732538769978 (+9 ms)Region opened successfully at 1732538769983 (+5 ms) 2024-11-25T12:46:09,983 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4e5a1bce44d223654427771878a045ee, disabling compactions & flushes 2024-11-25T12:46:09,983 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. 2024-11-25T12:46:09,983 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. 2024-11-25T12:46:09,983 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. after waiting 0 ms 2024-11-25T12:46:09,983 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. 2024-11-25T12:46:09,984 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. 2024-11-25T12:46:09,984 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4e5a1bce44d223654427771878a045ee: Waiting for close lock at 1732538769983Disabling compacts and flushes for region at 1732538769983Disabling writes for close at 1732538769983Writing region close event to WAL at 1732538769984 (+1 ms)Closed at 1732538769984 2024-11-25T12:46:09,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741878_1055 (size=95) 2024-11-25T12:46:09,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741878_1055 (size=95) 2024-11-25T12:46:09,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741878_1055 (size=95) 2024-11-25T12:46:09,991 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:09,991 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-46948328:(num 1732538769903) 2024-11-25T12:46:09,991 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:09,994 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:10,007 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994, exclude list is [], retry=0 2024-11-25T12:46:10,010 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:10,010 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:10,011 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:10,013 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 2024-11-25T12:46:10,013 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:10,013 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e5a1bce44d223654427771878a045ee, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:10,014 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:10,014 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,014 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,015 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,016 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName a 2024-11-25T12:46:10,016 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:10,017 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:10,017 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,017 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName b 2024-11-25T12:46:10,017 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:10,018 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:10,018 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,019 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName c 2024-11-25T12:46:10,019 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:10,019 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:10,019 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,020 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,022 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,022 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,023 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:10,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:10,025 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e5a1bce44d223654427771878a045ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67544960, jitterRate=0.0064983367919921875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:10,026 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e5a1bce44d223654427771878a045ee: Writing region info on filesystem at 1732538770014Initializing all the Stores at 1732538770015 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538770015Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538770015Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538770015Cleaning up temporary data from old regions at 1732538770022 (+7 ms)Region opened successfully at 1732538770026 (+4 ms) 2024-11-25T12:46:10,030 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732538770029/Put/seqid=0 2024-11-25T12:46:10,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741881_1058 (size=4826) 2024-11-25T12:46:10,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741881_1058 (size=4826) 2024-11-25T12:46:10,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741881_1058 (size=4826) 2024-11-25T12:46:10,440 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35793/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 4e5a1bce44d223654427771878a045ee/a 2024-11-25T12:46:10,452 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-25T12:46:10,452 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T12:46:10,453 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4e5a1bce44d223654427771878a045ee: 2024-11-25T12:46:10,456 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/a/d2500df141724011be640cae9b18a031_SeqId_3_ 2024-11-25T12:46:10,457 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35793/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 4e5a1bce44d223654427771878a045ee/a as hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/a/d2500df141724011be640cae9b18a031_SeqId_3_ - updating store file list. 2024-11-25T12:46:10,464 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d2500df141724011be640cae9b18a031_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-25T12:46:10,464 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/a/d2500df141724011be640cae9b18a031_SeqId_3_ into 4e5a1bce44d223654427771878a045ee/a 2024-11-25T12:46:10,464 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35793/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 4e5a1bce44d223654427771878a045ee/a (new location: hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/a/d2500df141724011be640cae9b18a031_SeqId_3_) 2024-11-25T12:46:10,511 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994, size=0 (0bytes) 2024-11-25T12:46:10,511 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 might be still open, length is 0 2024-11-25T12:46:10,511 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 2024-11-25T12:46:10,511 WARN [IPC Server handler 2 on default port 35793 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-25T12:46:10,512 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 after 1ms 2024-11-25T12:46:12,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:56400 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56400 dst: /127.0.0.1:34959 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34959 remote=/127.0.0.1:56400]. Total timeout mills is 60000, 58197 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:12,276 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:50208 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33377:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50208 dst: /127.0.0.1:33377 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:12,276 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:56704 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56704 dst: /127.0.0.1:35653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:12,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741880_1059 (size=473) 2024-11-25T12:46:12,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741880_1059 (size=473) 2024-11-25T12:46:14,512 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 after 4001ms 2024-11-25T12:46:14,516 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:14,516 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 took 4006ms 2024-11-25T12:46:14,518 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994; continuing. 2024-11-25T12:46:14,518 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 so closing down 2024-11-25T12:46:14,518 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:14,520 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732538769994.temp 2024-11-25T12:46:14,521 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005-wal.1732538769994.temp 2024-11-25T12:46:14,521 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741882_1060 (size=259) 2024-11-25T12:46:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741882_1060 (size=259) 2024-11-25T12:46:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741882_1060 (size=259) 2024-11-25T12:46:14,528 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005-wal.1732538769994.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:14,529 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005-wal.1732538769994.temp to hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005 2024-11-25T12:46:14,530 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T12:46:14,530 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994, journal: Splitting hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994, size=0 (0bytes) at 1732538770511Finishing writing output for hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 so closing down at 1732538774518 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005-wal.1732538769994.temp at 1732538774521 (+3 ms)3 split writer threads finished at 1732538774521Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005-wal.1732538769994.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732538774528 (+7 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005-wal.1732538769994.temp to hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005 at 1732538774529 (+1 ms)Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994, size=0, length=0, corrupted=false, cancelled=false at 1732538774530 (+1 ms) 2024-11-25T12:46:14,532 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538769994 2024-11-25T12:46:14,532 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005 2024-11-25T12:46:14,533 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:14,534 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:14,546 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538774535, exclude list is [], retry=0 2024-11-25T12:46:14,549 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:14,549 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:14,549 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:14,551 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538774535 2024-11-25T12:46:14,553 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:14,553 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e5a1bce44d223654427771878a045ee, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:14,554 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:14,554 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,554 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,555 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,556 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName a 2024-11-25T12:46:14,556 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:14,562 DEBUG [StoreFileOpener-4e5a1bce44d223654427771878a045ee-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d2500df141724011be640cae9b18a031_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-25T12:46:14,563 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/a/d2500df141724011be640cae9b18a031_SeqId_3_ 2024-11-25T12:46:14,563 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:14,563 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,564 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName b 2024-11-25T12:46:14,564 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:14,564 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:14,565 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,565 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5a1bce44d223654427771878a045ee columnFamilyName c 2024-11-25T12:46:14,565 DEBUG [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:14,566 INFO [StoreOpener-4e5a1bce44d223654427771878a045ee-1 {}] regionserver.HStore(327): Store=4e5a1bce44d223654427771878a045ee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:14,566 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,567 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,569 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005 2024-11-25T12:46:14,571 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:14,572 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005 2024-11-25T12:46:14,573 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4e5a1bce44d223654427771878a045ee 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-25T12:46:14,587 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/.tmp/a/decafedd285a495aa7e0371559d8c22b is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732538770470/Put/seqid=0 2024-11-25T12:46:14,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741884_1062 (size=5149) 2024-11-25T12:46:14,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741884_1062 (size=5149) 2024-11-25T12:46:14,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741884_1062 (size=5149) 2024-11-25T12:46:14,594 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/.tmp/a/decafedd285a495aa7e0371559d8c22b 2024-11-25T12:46:14,600 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/.tmp/a/decafedd285a495aa7e0371559d8c22b as hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/a/decafedd285a495aa7e0371559d8c22b 2024-11-25T12:46:14,605 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/a/decafedd285a495aa7e0371559d8c22b, entries=1, sequenceid=5, filesize=5.0 K 2024-11-25T12:46:14,606 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 4e5a1bce44d223654427771878a045ee in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-11-25T12:46:14,607 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/0000000000000000005 2024-11-25T12:46:14,608 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,608 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,609 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:14,610 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e5a1bce44d223654427771878a045ee 2024-11-25T12:46:14,612 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e5a1bce44d223654427771878a045ee/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-25T12:46:14,613 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e5a1bce44d223654427771878a045ee; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73179669, jitterRate=0.09046204388141632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:14,613 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e5a1bce44d223654427771878a045ee: Writing region info on filesystem at 1732538774554Initializing all the Stores at 1732538774555 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538774555Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538774555Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538774555Obtaining lock to block concurrent updates at 1732538774573 (+18 ms)Preparing flush snapshotting stores in 4e5a1bce44d223654427771878a045ee at 1732538774573Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732538774573Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. at 1732538774573Flushing 4e5a1bce44d223654427771878a045ee/a: creating writer at 1732538774573Flushing 4e5a1bce44d223654427771878a045ee/a: appending metadata at 1732538774586 (+13 ms)Flushing 4e5a1bce44d223654427771878a045ee/a: closing flushed file at 1732538774586Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55784d4e: reopening flushed file at 1732538774599 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 4e5a1bce44d223654427771878a045ee in 34ms, sequenceid=5, compaction requested=false; wal=null at 1732538774606 (+7 ms)Cleaning up temporary data from old regions at 1732538774608 (+2 ms)Region opened successfully at 1732538774613 (+5 ms) 2024-11-25T12:46:14,617 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4e5a1bce44d223654427771878a045ee, disabling compactions & flushes 2024-11-25T12:46:14,617 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. 2024-11-25T12:46:14,618 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. 2024-11-25T12:46:14,618 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. after waiting 0 ms 2024-11-25T12:46:14,618 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. 2024-11-25T12:46:14,618 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732538769893.4e5a1bce44d223654427771878a045ee. 2024-11-25T12:46:14,619 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4e5a1bce44d223654427771878a045ee: Waiting for close lock at 1732538774617Disabling compacts and flushes for region at 1732538774617Disabling writes for close at 1732538774618 (+1 ms)Writing region close event to WAL at 1732538774618Closed at 1732538774618 2024-11-25T12:46:14,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741883_1061 (size=95) 2024-11-25T12:46:14,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741883_1061 (size=95) 2024-11-25T12:46:14,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741883_1061 (size=95) 2024-11-25T12:46:14,623 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:14,623 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538774535) 2024-11-25T12:46:14,637 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=410 (was 405) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2097936468_22 at /127.0.0.1:33132 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2097936468_22 at /127.0.0.1:49642 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40159 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:35793 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35793 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:40159 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2097936468_22 at /127.0.0.1:40600 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1005 (was 945) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=660 (was 639) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1329 (was 1772) 2024-11-25T12:46:14,648 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=410, OpenFileDescriptor=1005, MaxFileDescriptor=1048576, SystemLoadAverage=660, ProcessCount=11, AvailableMemoryMB=1328 2024-11-25T12:46:14,663 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:14,667 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:46:14,671 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,36927,1732538755691 2024-11-25T12:46:14,674 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27482298 2024-11-25T12:46:14,675 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:46:14,678 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:46:14,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:46:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-25T12:46:14,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:46:14,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-25T12:46:14,695 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:14,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:46:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:46:14,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741885_1063 (size=694) 2024-11-25T12:46:14,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741885_1063 (size=694) 2024-11-25T12:46:14,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741885_1063 (size=694) 2024-11-25T12:46:14,720 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c6d3a9a4a6e575e3ff59df3723af337c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7 2024-11-25T12:46:14,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741886_1064 (size=77) 2024-11-25T12:46:14,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741886_1064 (size=77) 2024-11-25T12:46:14,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741886_1064 (size=77) 2024-11-25T12:46:14,732 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:14,732 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing c6d3a9a4a6e575e3ff59df3723af337c, disabling compactions & flushes 2024-11-25T12:46:14,732 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:14,732 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:14,732 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. after waiting 0 ms 2024-11-25T12:46:14,732 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:14,732 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:14,732 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for c6d3a9a4a6e575e3ff59df3723af337c: Waiting for close lock at 1732538774732Disabling compacts and flushes for region at 1732538774732Disabling writes for close at 1732538774732Writing region close event to WAL at 1732538774732Closed at 1732538774732 2024-11-25T12:46:14,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:46:14,740 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732538774734"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732538774734"}]},"ts":"1732538774734"} 2024-11-25T12:46:14,743 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:46:14,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:46:14,747 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732538774745"}]},"ts":"1732538774745"} 2024-11-25T12:46:14,750 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-25T12:46:14,751 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1ff8009e463e=0} racks are {/default-rack=0} 2024-11-25T12:46:14,752 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T12:46:14,753 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T12:46:14,753 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T12:46:14,753 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T12:46:14,753 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T12:46:14,753 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T12:46:14,753 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T12:46:14,753 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T12:46:14,753 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T12:46:14,753 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T12:46:14,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN}] 2024-11-25T12:46:14,757 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN 2024-11-25T12:46:14,759 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN; state=OFFLINE, location=1ff8009e463e,38825,1732538756840; forceNewPlan=false, retain=false 2024-11-25T12:46:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:46:14,911 INFO [1ff8009e463e:36927 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T12:46:14,912 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=OPENING, regionLocation=1ff8009e463e,38825,1732538756840 2024-11-25T12:46:14,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN because future has completed 2024-11-25T12:46:14,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840}] 2024-11-25T12:46:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:46:15,078 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,079 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c6d3a9a4a6e575e3ff59df3723af337c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:15,080 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,080 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:15,080 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,080 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,083 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,086 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf1 2024-11-25T12:46:15,087 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:15,088 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:15,088 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,091 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf2 2024-11-25T12:46:15,091 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:15,091 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:15,092 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,092 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,093 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,094 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,094 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,095 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T12:46:15,097 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,099 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:15,100 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c6d3a9a4a6e575e3ff59df3723af337c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71703866, jitterRate=0.06847086548805237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T12:46:15,100 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,100 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c6d3a9a4a6e575e3ff59df3723af337c: Running coprocessor pre-open hook at 1732538775080Writing region info on filesystem at 1732538775080Initializing all the Stores at 1732538775082 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538775082Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538775083 (+1 ms)Cleaning up temporary data from old regions at 1732538775094 (+11 ms)Running coprocessor post-open hooks at 1732538775100 (+6 ms)Region opened successfully at 1732538775100 2024-11-25T12:46:15,102 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., pid=6, masterSystemTime=1732538775070 2024-11-25T12:46:15,105 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,105 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,106 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,38825,1732538756840 2024-11-25T12:46:15,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840 because future has completed 2024-11-25T12:46:15,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T12:46:15,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840 in 195 msec 2024-11-25T12:46:15,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T12:46:15,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN in 361 msec 2024-11-25T12:46:15,121 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:46:15,121 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732538775121"}]},"ts":"1732538775121"} 2024-11-25T12:46:15,125 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-25T12:46:15,127 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:46:15,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 443 msec 2024-11-25T12:46:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:46:15,333 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-25T12:46:15,333 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-25T12:46:15,335 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T12:46:15,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-25T12:46:15,341 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T12:46:15,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-25T12:46:15,355 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=2] 2024-11-25T12:46:15,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=c6d3a9a4a6e575e3ff59df3723af337c, source=1ff8009e463e,38825,1732538756840, destination=1ff8009e463e,43295,1732538756657, warming up region on 1ff8009e463e,43295,1732538756657 2024-11-25T12:46:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:46:15,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=c6d3a9a4a6e575e3ff59df3723af337c, source=1ff8009e463e,38825,1732538756840, destination=1ff8009e463e,43295,1732538756657, running balancer 2024-11-25T12:46:15,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54469, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:46:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE 2024-11-25T12:46:15,379 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE 2024-11-25T12:46:15,381 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=CLOSING, regionLocation=1ff8009e463e,38825,1732538756840 2024-11-25T12:46:15,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(7855): Warmup {ENCODED => c6d3a9a4a6e575e3ff59df3723af337c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:15,385 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE because future has completed 2024-11-25T12:46:15,385 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:46:15,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840}] 2024-11-25T12:46:15,386 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf1 2024-11-25T12:46:15,386 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:15,386 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:15,386 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,387 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf2 2024-11-25T12:46:15,387 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:15,388 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(1722): Closing c6d3a9a4a6e575e3ff59df3723af337c, disabling compactions & flushes 2024-11-25T12:46:15,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. after waiting 0 ms 2024-11-25T12:46:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] regionserver.HRegion(1676): Region close journal for c6d3a9a4a6e575e3ff59df3723af337c: Waiting for close lock at 1732538775388Disabling compacts and flushes for region at 1732538775388Disabling writes for close at 1732538775388Writing region close event to WAL at 1732538775389 (+1 ms)Closed at 1732538775389 2024-11-25T12:46:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-25T12:46:15,542 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,543 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:46:15,543 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing c6d3a9a4a6e575e3ff59df3723af337c, disabling compactions & flushes 2024-11-25T12:46:15,543 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,543 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,543 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. after waiting 0 ms 2024-11-25T12:46:15,543 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,543 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing c6d3a9a4a6e575e3ff59df3723af337c 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-25T12:46:15,565 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf1/b7fc7bf0c4764100aff552b084425762 is 35, key is r1/cf1:q/1732538775358/Put/seqid=0 2024-11-25T12:46:15,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741887_1065 (size=4783) 2024-11-25T12:46:15,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741887_1065 (size=4783) 2024-11-25T12:46:15,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741887_1065 (size=4783) 2024-11-25T12:46:15,574 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf1/b7fc7bf0c4764100aff552b084425762 2024-11-25T12:46:15,583 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf1/b7fc7bf0c4764100aff552b084425762 as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/b7fc7bf0c4764100aff552b084425762 2024-11-25T12:46:15,589 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/b7fc7bf0c4764100aff552b084425762, entries=1, sequenceid=5, filesize=4.7 K 2024-11-25T12:46:15,590 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for c6d3a9a4a6e575e3ff59df3723af337c in 47ms, sequenceid=5, compaction requested=false 2024-11-25T12:46:15,590 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-25T12:46:15,596 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-25T12:46:15,598 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,598 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for c6d3a9a4a6e575e3ff59df3723af337c: Waiting for close lock at 1732538775543Running coprocessor pre-close hooks at 1732538775543Disabling compacts and flushes for region at 1732538775543Disabling writes for close at 1732538775543Obtaining lock to block concurrent updates at 1732538775543Preparing flush snapshotting stores in c6d3a9a4a6e575e3ff59df3723af337c at 1732538775543Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732538775544 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. at 1732538775544Flushing c6d3a9a4a6e575e3ff59df3723af337c/cf1: creating writer at 1732538775545 (+1 ms)Flushing c6d3a9a4a6e575e3ff59df3723af337c/cf1: appending metadata at 1732538775565 (+20 ms)Flushing c6d3a9a4a6e575e3ff59df3723af337c/cf1: closing flushed file at 1732538775565Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49bafada: reopening flushed file at 1732538775581 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for c6d3a9a4a6e575e3ff59df3723af337c in 47ms, sequenceid=5, compaction requested=false at 1732538775590 (+9 ms)Writing region close event to WAL at 1732538775591 (+1 ms)Running coprocessor post-close hooks at 1732538775596 (+5 ms)Closed at 1732538775598 (+2 ms) 2024-11-25T12:46:15,599 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding c6d3a9a4a6e575e3ff59df3723af337c move to 1ff8009e463e,43295,1732538756657 record at close sequenceid=5 2024-11-25T12:46:15,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=CLOSED 2024-11-25T12:46:15,604 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840 because future has completed 2024-11-25T12:46:15,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T12:46:15,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840 in 222 msec 2024-11-25T12:46:15,613 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE; state=CLOSED, location=1ff8009e463e,43295,1732538756657; forceNewPlan=false, retain=false 2024-11-25T12:46:15,764 INFO [1ff8009e463e:36927 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T12:46:15,764 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=OPENING, regionLocation=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:15,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE because future has completed 2024-11-25T12:46:15,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657}] 2024-11-25T12:46:15,925 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,925 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c6d3a9a4a6e575e3ff59df3723af337c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:15,926 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,926 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:15,926 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,926 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,928 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,929 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf1 2024-11-25T12:46:15,929 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:15,937 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/b7fc7bf0c4764100aff552b084425762 2024-11-25T12:46:15,937 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:15,938 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,939 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf2 2024-11-25T12:46:15,939 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:15,940 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:15,940 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,941 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,942 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,943 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,943 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,944 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T12:46:15,946 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,947 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened c6d3a9a4a6e575e3ff59df3723af337c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66603369, jitterRate=-0.007532462477684021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T12:46:15,947 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:15,948 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c6d3a9a4a6e575e3ff59df3723af337c: Running coprocessor pre-open hook at 1732538775926Writing region info on filesystem at 1732538775926Initializing all the Stores at 1732538775927 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538775927Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538775927Cleaning up temporary data from old regions at 1732538775943 (+16 ms)Running coprocessor post-open hooks at 1732538775947 (+4 ms)Region opened successfully at 1732538775948 (+1 ms) 2024-11-25T12:46:15,949 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., pid=9, masterSystemTime=1732538775921 2024-11-25T12:46:15,952 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,952 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:15,953 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=OPEN, openSeqNum=9, regionLocation=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:15,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657 because future has completed 2024-11-25T12:46:15,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-25T12:46:15,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657 in 194 msec 2024-11-25T12:46:15,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE in 589 msec 2024-11-25T12:46:15,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:46:15,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:46:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:48978 deadline: 1732538835992, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=43295 startCode=1732538756657. As of locationSeqNum=5. 2024-11-25T12:46:16,031 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=43295 startCode=1732538756657. As of locationSeqNum=5. 2024-11-25T12:46:16,032 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=43295 startCode=1732538756657. As of locationSeqNum=5. 2024-11-25T12:46:16,032 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,43295,1732538756657, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=43295 startCode=1732538756657. As of locationSeqNum=5. 2024-11-25T12:46:16,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-25T12:46:16,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:16,127 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-25T12:46:16,127 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:16,128 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-25T12:46:16,128 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:16,142 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:46:16,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:46:16,157 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c6d3a9a4a6e575e3ff59df3723af337c 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-25T12:46:16,178 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf1/2547b763746147198b4b3db4d5fdc7a6 is 29, key is r1/cf1:/1732538776147/DeleteFamily/seqid=0 2024-11-25T12:46:16,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741888_1066 (size=4906) 2024-11-25T12:46:16,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741888_1066 (size=4906) 2024-11-25T12:46:16,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741888_1066 (size=4906) 2024-11-25T12:46:16,594 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf1/2547b763746147198b4b3db4d5fdc7a6 2024-11-25T12:46:16,603 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2547b763746147198b4b3db4d5fdc7a6 2024-11-25T12:46:16,617 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf2/c7d78f6928914ef1820fc71d3c6120c8 is 29, key is r1/cf2:/1732538776147/DeleteFamily/seqid=0 2024-11-25T12:46:16,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741889_1067 (size=4906) 2024-11-25T12:46:16,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741889_1067 (size=4906) 2024-11-25T12:46:16,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741889_1067 (size=4906) 2024-11-25T12:46:16,624 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf2/c7d78f6928914ef1820fc71d3c6120c8 2024-11-25T12:46:16,631 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7d78f6928914ef1820fc71d3c6120c8 2024-11-25T12:46:16,632 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf1/2547b763746147198b4b3db4d5fdc7a6 as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/2547b763746147198b4b3db4d5fdc7a6 2024-11-25T12:46:16,638 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2547b763746147198b4b3db4d5fdc7a6 2024-11-25T12:46:16,639 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/2547b763746147198b4b3db4d5fdc7a6, entries=1, sequenceid=12, filesize=4.8 K 2024-11-25T12:46:16,640 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf2/c7d78f6928914ef1820fc71d3c6120c8 as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/c7d78f6928914ef1820fc71d3c6120c8 2024-11-25T12:46:16,645 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7d78f6928914ef1820fc71d3c6120c8 2024-11-25T12:46:16,646 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/c7d78f6928914ef1820fc71d3c6120c8, entries=1, sequenceid=12, filesize=4.8 K 2024-11-25T12:46:16,647 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for c6d3a9a4a6e575e3ff59df3723af337c in 490ms, sequenceid=12, compaction requested=false 2024-11-25T12:46:16,647 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c6d3a9a4a6e575e3ff59df3723af337c: 2024-11-25T12:46:16,649 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T12:46:16,651 DEBUG [Time-limited test {}] regionserver.HStore(1541): c6d3a9a4a6e575e3ff59df3723af337c/cf1 is initiating major compaction (all files) 2024-11-25T12:46:16,651 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:46:16,651 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:16,652 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c6d3a9a4a6e575e3ff59df3723af337c/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,652 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/b7fc7bf0c4764100aff552b084425762, hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/2547b763746147198b4b3db4d5fdc7a6] into tmpdir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp, totalSize=9.5 K 2024-11-25T12:46:16,653 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b7fc7bf0c4764100aff552b084425762, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732538775358 2024-11-25T12:46:16,654 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2547b763746147198b4b3db4d5fdc7a6, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-25T12:46:16,666 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): c6d3a9a4a6e575e3ff59df3723af337c#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:46:16,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741890_1068 (size=4626) 2024-11-25T12:46:16,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741890_1068 (size=4626) 2024-11-25T12:46:16,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741890_1068 (size=4626) 2024-11-25T12:46:16,682 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf1/969e437bfc3a4ed2996bc9f6d1618a79 as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/969e437bfc3a4ed2996bc9f6d1618a79 2024-11-25T12:46:16,696 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in c6d3a9a4a6e575e3ff59df3723af337c/cf1 of c6d3a9a4a6e575e3ff59df3723af337c into 969e437bfc3a4ed2996bc9f6d1618a79(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:46:16,696 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c6d3a9a4a6e575e3ff59df3723af337c: 2024-11-25T12:46:16,696 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-25T12:46:16,697 DEBUG [Time-limited test {}] regionserver.HStore(1541): c6d3a9a4a6e575e3ff59df3723af337c/cf2 is initiating major compaction (all files) 2024-11-25T12:46:16,697 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:46:16,697 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:46:16,697 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c6d3a9a4a6e575e3ff59df3723af337c/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,697 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/c7d78f6928914ef1820fc71d3c6120c8] into tmpdir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp, totalSize=4.8 K 2024-11-25T12:46:16,698 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c7d78f6928914ef1820fc71d3c6120c8, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-25T12:46:16,704 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): c6d3a9a4a6e575e3ff59df3723af337c#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:46:16,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741891_1069 (size=4592) 2024-11-25T12:46:16,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741891_1069 (size=4592) 2024-11-25T12:46:16,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741891_1069 (size=4592) 2024-11-25T12:46:16,719 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/.tmp/cf2/b3f4dff6c2784ffbbe1977a0aee9fd91 as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/b3f4dff6c2784ffbbe1977a0aee9fd91 2024-11-25T12:46:16,727 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in c6d3a9a4a6e575e3ff59df3723af337c/cf2 of c6d3a9a4a6e575e3ff59df3723af337c into b3f4dff6c2784ffbbe1977a0aee9fd91(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:46:16,727 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c6d3a9a4a6e575e3ff59df3723af337c: 2024-11-25T12:46:16,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=c6d3a9a4a6e575e3ff59df3723af337c, source=1ff8009e463e,43295,1732538756657, destination=1ff8009e463e,38825,1732538756840, warming up region on 1ff8009e463e,38825,1732538756840 2024-11-25T12:46:16,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=c6d3a9a4a6e575e3ff59df3723af337c, source=1ff8009e463e,43295,1732538756657, destination=1ff8009e463e,38825,1732538756840, running balancer 2024-11-25T12:46:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE 2024-11-25T12:46:16,734 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE 2024-11-25T12:46:16,735 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=CLOSING, regionLocation=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:16,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(7855): Warmup {ENCODED => c6d3a9a4a6e575e3ff59df3723af337c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:16,737 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:16,738 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf1 2024-11-25T12:46:16,738 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:16,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE because future has completed 2024-11-25T12:46:16,739 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:46:16,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657}] 2024-11-25T12:46:16,746 INFO [StoreFileOpener-c6d3a9a4a6e575e3ff59df3723af337c-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2547b763746147198b4b3db4d5fdc7a6 2024-11-25T12:46:16,747 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/2547b763746147198b4b3db4d5fdc7a6 2024-11-25T12:46:16,752 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/969e437bfc3a4ed2996bc9f6d1618a79 2024-11-25T12:46:16,760 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/b7fc7bf0c4764100aff552b084425762 2024-11-25T12:46:16,760 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:16,760 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:16,761 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf2 2024-11-25T12:46:16,761 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:16,770 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/b3f4dff6c2784ffbbe1977a0aee9fd91 2024-11-25T12:46:16,776 INFO [StoreFileOpener-c6d3a9a4a6e575e3ff59df3723af337c-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7d78f6928914ef1820fc71d3c6120c8 2024-11-25T12:46:16,776 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/c7d78f6928914ef1820fc71d3c6120c8 2024-11-25T12:46:16,776 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(1722): Closing c6d3a9a4a6e575e3ff59df3723af337c, disabling compactions & flushes 2024-11-25T12:46:16,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. after waiting 0 ms 2024-11-25T12:46:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38825 {}] regionserver.HRegion(1676): Region close journal for c6d3a9a4a6e575e3ff59df3723af337c: Waiting for close lock at 1732538776777Disabling compacts and flushes for region at 1732538776777Disabling writes for close at 1732538776777Writing region close event to WAL at 1732538776778 (+1 ms)Closed at 1732538776778 2024-11-25T12:46:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-25T12:46:16,894 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:16,894 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:46:16,894 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing c6d3a9a4a6e575e3ff59df3723af337c, disabling compactions & flushes 2024-11-25T12:46:16,894 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,894 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,894 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. after waiting 0 ms 2024-11-25T12:46:16,894 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,895 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/b7fc7bf0c4764100aff552b084425762, hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/2547b763746147198b4b3db4d5fdc7a6] to archive 2024-11-25T12:46:16,898 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:46:16,902 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/b7fc7bf0c4764100aff552b084425762 to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/b7fc7bf0c4764100aff552b084425762 2024-11-25T12:46:16,905 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/2547b763746147198b4b3db4d5fdc7a6 to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/2547b763746147198b4b3db4d5fdc7a6 2024-11-25T12:46:16,919 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/c7d78f6928914ef1820fc71d3c6120c8] to archive 2024-11-25T12:46:16,920 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:46:16,923 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/c7d78f6928914ef1820fc71d3c6120c8 to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/c7d78f6928914ef1820fc71d3c6120c8 2024-11-25T12:46:16,929 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-25T12:46:16,930 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:16,930 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for c6d3a9a4a6e575e3ff59df3723af337c: Waiting for close lock at 1732538776894Running coprocessor pre-close hooks at 1732538776894Disabling compacts and flushes for region at 1732538776894Disabling writes for close at 1732538776894Writing region close event to WAL at 1732538776924 (+30 ms)Running coprocessor post-close hooks at 1732538776930 (+6 ms)Closed at 1732538776930 2024-11-25T12:46:16,930 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding c6d3a9a4a6e575e3ff59df3723af337c move to 1ff8009e463e,38825,1732538756840 record at close sequenceid=12 2024-11-25T12:46:16,933 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:16,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=CLOSED 2024-11-25T12:46:16,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657 because future has completed 2024-11-25T12:46:16,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T12:46:16,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657 in 200 msec 2024-11-25T12:46:16,943 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE; state=CLOSED, location=1ff8009e463e,38825,1732538756840; forceNewPlan=false, retain=false 2024-11-25T12:46:17,094 INFO [1ff8009e463e:36927 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T12:46:17,094 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=OPENING, regionLocation=1ff8009e463e,38825,1732538756840 2024-11-25T12:46:17,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE because future has completed 2024-11-25T12:46:17,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840}] 2024-11-25T12:46:17,257 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:17,257 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => c6d3a9a4a6e575e3ff59df3723af337c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:17,257 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,258 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:17,258 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,258 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,259 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,261 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf1 2024-11-25T12:46:17,261 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:17,269 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/969e437bfc3a4ed2996bc9f6d1618a79 2024-11-25T12:46:17,269 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:17,269 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,271 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf2 2024-11-25T12:46:17,271 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:17,278 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/b3f4dff6c2784ffbbe1977a0aee9fd91 2024-11-25T12:46:17,278 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:17,278 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,279 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,280 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,282 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,283 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,283 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T12:46:17,285 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,286 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened c6d3a9a4a6e575e3ff59df3723af337c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66991880, jitterRate=-0.0017431974411010742}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T12:46:17,286 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,287 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for c6d3a9a4a6e575e3ff59df3723af337c: Running coprocessor pre-open hook at 1732538777258Writing region info on filesystem at 1732538777258Initializing all the Stores at 1732538777259 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538777259Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538777259Cleaning up temporary data from old regions at 1732538777283 (+24 ms)Running coprocessor post-open hooks at 1732538777286 (+3 ms)Region opened successfully at 1732538777287 (+1 ms) 2024-11-25T12:46:17,291 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., pid=12, masterSystemTime=1732538777253 2024-11-25T12:46:17,293 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:17,293 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:17,294 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=OPEN, openSeqNum=18, regionLocation=1ff8009e463e,38825,1732538756840 2024-11-25T12:46:17,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840 because future has completed 2024-11-25T12:46:17,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-25T12:46:17,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,38825,1732538756840 in 201 msec 2024-11-25T12:46:17,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, REOPEN/MOVE in 574 msec 2024-11-25T12:46:17,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:46:17,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:46:17,344 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 1ff8009e463e,38825,1732538756840: testing ***** 2024-11-25T12:46:17,344 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-25T12:46:17,346 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-25T12:46:17,348 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-25T12:46:17,355 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-25T12:46:17,358 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-25T12:46:17,386 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 328602416 }, "NonHeapMemoryUsage": { "committed": 171507712, "init": 7667712, "max": -1, "used": 168973760 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1ff8009e463e", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 46, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 7, "ProcessCallTime_95th_percentile": 10, "ProcessCallTime_98th_percentile": 46, "ProcessCallTime_99th_percentile": 46, "ProcessCallTime_99.9th_percentile": 46, "ProcessCallTime_TimeRangeCount_0-1": 7, "ProcessCallTime_TimeRangeCount_1-3": 17, "ProcessCallTime_TimeRangeCount_3-10": 6, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 46, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 7, "TotalCallTime_95th_percentile": 10, "TotalCallTime_98th_percentile": 46, "TotalCallTime_99th_percentile": 46, "TotalCallTime_99.9th_percentile": 46, "TotalCallTime_TimeRangeCount_0-1": 6, "TotalCallTime_TimeRangeCount_1-3": 17, "TotalCallTime_TimeRangeCount_3-10": 7, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1ff8009e463e", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:51952", "tag.serverName": "1ff8009e463e,43295,1732538756657", "tag.clusterId": "c2a5e957-eae0-4d39-83b1-2894938e9454", "tag.Context": "regionserver", "tag.Hostname": "1ff8009e463e", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732538756657, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 3, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 1, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 30, "MajorCompactionTime_max": 42, "MajorCompactionTime_mean": 36, "MajorCompactionTime_25th_percentile": 30, "MajorCompactionTime_median": 31, "MajorCompactionTime_75th_percentile": 42, "MajorCompactionTime_90th_percentile": 42, "MajorCompactionTime_95th_percentile": 42, "MajorCompactionTime_98th_percentile": 42, "MajorCompactionTime_99th_percentile": 42, "MajorCompactionTime_99.9th_percentile": 42, "MajorCompactionTime_TimeRangeCount_10-30": 1, "MajorCompactionTime_TimeRangeCount_30-100": 1, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 3, "Delete_max": 3, "Delete_mean": 3, "Delete_25th_percentile": 3, "Delete_median": 3, "Delete_75th_percentile": 3, "Delete_90th_percentile": 3, "Delete_95th_percentile": 3, "Delete_98th_percentile": 3, "Delete_99th_percentile": 3, "Delete_99.9th_percentile": 3, "Delete_TimeRangeCount_1-3": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 30, "CompactionTime_max": 42, "CompactionTime_mean": 36, "CompactionTime_25th_percentile": 30, "CompactionTime_median": 31, "CompactionTime_75th_percentile": 42, "CompactionTime_90th_percentile": 42, "CompactionTime_95th_percentile": 42, "CompactionTime_98th_percentile": 42, "CompactionTime_99th_percentile": 42, "CompactionTime_99.9th_percentile": 42, "CompactionTime_TimeRangeCount_10-30": 1, "CompactionTime_TimeRangeCount_30-100": 1, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 490, "FlushTime_max": 490, "FlushTime_mean": 490, "FlushTime_25th_percentile": 490, "FlushTime_median": 490, "FlushTime_75th_percentile": 490, "FlushTime_90th_percentile": 490, "FlushTime_95th_percentile": 490, "FlushTime_98th_percentile": 490, "FlushTime_99th_percentile": 490, "FlushTime_99.9th_percentile": 490, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 1, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-25T12:46:17,392 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36927 {}] master.MasterRpcServices(700): 1ff8009e463e,38825,1732538756840 reported a fatal error: ***** ABORTING region server 1ff8009e463e,38825,1732538756840: testing ***** 2024-11-25T12:46:17,394 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ff8009e463e,38825,1732538756840' ***** 2024-11-25T12:46:17,394 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-25T12:46:17,395 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T12:46:17,395 INFO [RS:1;1ff8009e463e:38825 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-25T12:46:17,395 INFO [RS:1;1ff8009e463e:38825 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-25T12:46:17,395 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(3091): Received CLOSE for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,395 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(956): aborting server 1ff8009e463e,38825,1732538756840 2024-11-25T12:46:17,396 INFO [RS:1;1ff8009e463e:38825 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:46:17,396 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c6d3a9a4a6e575e3ff59df3723af337c, disabling compactions & flushes 2024-11-25T12:46:17,396 INFO [RS:1;1ff8009e463e:38825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1ff8009e463e:38825. 2024-11-25T12:46:17,396 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:17,396 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:17,396 DEBUG [RS:1;1ff8009e463e:38825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:46:17,396 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. after waiting 0 ms 2024-11-25T12:46:17,396 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:17,396 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T12:46:17,396 DEBUG [RS:1;1ff8009e463e:38825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:17,397 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T12:46:17,397 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T12:46:17,397 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T12:46:17,397 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T12:46:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:46778 deadline: 1732538837399, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=38825 startCode=1732538756840. As of locationSeqNum=12. 2024-11-25T12:46:17,401 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,43295,1732538756657, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,43295,1732538756657, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=38825 startCode=1732538756840. As of locationSeqNum=12. 2024-11-25T12:46:17,401 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,43295,1732538756657, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=38825 startCode=1732538756840. As of locationSeqNum=12. 2024-11-25T12:46:17,401 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,43295,1732538756657, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=38825 startCode=1732538756840. As of locationSeqNum=12. 2024-11-25T12:46:17,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,38825,1732538756840 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:17,404 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,38825,1732538756840 aborting 2024-11-25T12:46:17,404 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,38825,1732538756840 aborting 2024-11-25T12:46:17,404 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 from cache 2024-11-25T12:46:17,407 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T12:46:17,407 DEBUG [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, c6d3a9a4a6e575e3ff59df3723af337c=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.} 2024-11-25T12:46:17,408 DEBUG [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:17,408 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T12:46:17,408 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T12:46:17,409 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T12:46:17,409 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T12:46:17,409 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T12:46:17,418 ERROR [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-25T12:46:17,420 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T12:46:17,420 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T12:46:17,420 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732538777408Running coprocessor pre-close hooks at 1732538777408Disabling compacts and flushes for region at 1732538777408Disabling writes for close at 1732538777409 (+1 ms)Writing region close event to WAL at 1732538777418 (+9 ms)Running coprocessor post-close hooks at 1732538777419 (+1 ms)Closed at 1732538777420 (+1 ms) 2024-11-25T12:46:17,420 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T12:46:17,423 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:17,424 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c6d3a9a4a6e575e3ff59df3723af337c: Waiting for close lock at 1732538777396Running coprocessor pre-close hooks at 1732538777396Disabling compacts and flushes for region at 1732538777396Disabling writes for close at 1732538777396Writing region close event to WAL at 1732538777423 (+27 ms)Running coprocessor post-close hooks at 1732538777423Closed at 1732538777423 2024-11-25T12:46:17,424 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:17,453 INFO [regionserver/1ff8009e463e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:46:17,512 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:46:17,514 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1] 2024-11-25T12:46:17,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,38825,1732538756840 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:17,516 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,38825,1732538756840 aborting 2024-11-25T12:46:17,516 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,38825,1732538756840 aborting 2024-11-25T12:46:17,517 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 from cache 2024-11-25T12:46:17,608 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(976): stopping server 1ff8009e463e,38825,1732538756840; all regions closed. 2024-11-25T12:46:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741836_1012 (size=3561) 2024-11-25T12:46:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741836_1012 (size=3561) 2024-11-25T12:46:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741836_1012 (size=3561) 2024-11-25T12:46:17,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741835_1011 (size=1406) 2024-11-25T12:46:17,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741835_1011 (size=1406) 2024-11-25T12:46:17,616 DEBUG [RS:1;1ff8009e463e:38825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:17,616 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:46:17,616 INFO [RS:1;1ff8009e463e:38825 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:46:17,617 INFO [RS:1;1ff8009e463e:38825 {}] hbase.ChoreService(370): Chore service for: regionserver/1ff8009e463e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T12:46:17,617 INFO [RS:1;1ff8009e463e:38825 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:46:17,617 INFO [regionserver/1ff8009e463e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:46:17,618 INFO [RS:1;1ff8009e463e:38825 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38825 2024-11-25T12:46:17,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:46:17,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ff8009e463e,38825,1732538756840 2024-11-25T12:46:17,637 INFO [RS:1;1ff8009e463e:38825 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:46:17,639 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ff8009e463e,38825,1732538756840] 2024-11-25T12:46:17,654 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ff8009e463e,38825,1732538756840 already deleted, retry=false 2024-11-25T12:46:17,654 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1ff8009e463e,38825,1732538756840 on 1ff8009e463e,36927,1732538755691 2024-11-25T12:46:17,661 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1ff8009e463e,38825,1732538756840, splitWal=true, meta=true 2024-11-25T12:46:17,664 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 1ff8009e463e,38825,1732538756840 (carryingMeta=true) 1ff8009e463e,38825,1732538756840/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4552327d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T12:46:17,665 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1ff8009e463e,38825,1732538756840, splitWal=true, meta=true 2024-11-25T12:46:17,666 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 1ff8009e463e,38825,1732538756840, splitWal=true, meta=true, isMeta: true 2024-11-25T12:46:17,668 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting 2024-11-25T12:46:17,670 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1ff8009e463e,38825,1732538756840 WAL count=1, meta=true 2024-11-25T12:46:17,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta}] 2024-11-25T12:46:17,678 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:17,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, worker=1ff8009e463e,43295,1732538756657}] 2024-11-25T12:46:17,721 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:46:17,723 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1] 2024-11-25T12:46:17,724 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1ff8009e463e:38825 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ff8009e463e/172.17.0.3:38825 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:17,725 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1, error=java.net.ConnectException: Call to address=1ff8009e463e:38825 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ff8009e463e/172.17.0.3:38825 2024-11-25T12:46:17,725 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-25T12:46:17,725 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 from cache 2024-11-25T12:46:17,726 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 1ff8009e463e:38825 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ff8009e463e/172.17.0.3:38825 2024-11-25T12:46:17,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:46:17,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38825-0x101735117c30002, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:46:17,747 INFO [RS:1;1ff8009e463e:38825 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:46:17,747 INFO [RS:1;1ff8009e463e:38825 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ff8009e463e,38825,1732538756840; zookeeper connection closed. 2024-11-25T12:46:17,747 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@606a5d6d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@606a5d6d 2024-11-25T12:46:17,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43295 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-25T12:46:17,864 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, size=3.5 K (3561bytes) 2024-11-25T12:46:17,864 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta 2024-11-25T12:46:17,865 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta after 1ms 2024-11-25T12:46:17,867 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:17,868 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta took 4ms 2024-11-25T12:46:17,875 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-25T12:46:17,876 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta so closing down 2024-11-25T12:46:17,876 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:17,878 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta.temp 2024-11-25T12:46:17,879 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta.temp 2024-11-25T12:46:17,880 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:17,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741892_1070 (size=3346) 2024-11-25T12:46:17,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741892_1070 (size=3346) 2024-11-25T12:46:17,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741892_1070 (size=3346) 2024-11-25T12:46:17,895 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:17,896 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta.temp to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-25T12:46:17,897 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 28 ms; skipped=1; WAL=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false 2024-11-25T12:46:17,897 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, journal: Splitting hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, size=3.5 K (3561bytes) at 1732538777864Finishing writing output for hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta so closing down at 1732538777876 (+12 ms)Creating recovered edits writer path=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta.temp at 1732538777879 (+3 ms)3 split writer threads finished at 1732538777881 (+2 ms)Closed recovered edits writer path=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732538777895 (+14 ms)Rename recovered edits hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta.temp to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732538777896 (+1 ms)Processed 16 edits across 1 Regions in 28 ms; skipped=1; WAL=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false at 1732538777897 (+1 ms) 2024-11-25T12:46:17,897 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta 2024-11-25T12:46:17,898 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-25T12:46:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-25T12:46:17,906 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs 2024-11-25T12:46:17,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-25T12:46:17,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, worker=1ff8009e463e,43295,1732538756657 in 226 msec 2024-11-25T12:46:17,911 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:17,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T12:46:17,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 1ff8009e463e%2C38825%2C1732538756840.meta.1732538760463.meta, worker=1ff8009e463e,43295,1732538756657 in 240 msec 2024-11-25T12:46:17,916 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting dir is empty, no logs to split. 2024-11-25T12:46:17,916 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1ff8009e463e,38825,1732538756840 WAL count=0, meta=true 2024-11-25T12:46:17,916 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 1ff8009e463e,38825,1732538756840 WAL splitting is done? wals=0, meta=true 2024-11-25T12:46:17,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T12:46:17,920 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:46:17,921 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T12:46:18,032 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:46:18,034 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1] 2024-11-25T12:46:18,034 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ff8009e463e:38825 this server is in the failed servers list 2024-11-25T12:46:18,035 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ff8009e463e:38825 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ff8009e463e:38825 2024-11-25T12:46:18,035 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ff8009e463e:38825 2024-11-25T12:46:18,035 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 from cache 2024-11-25T12:46:18,075 DEBUG [1ff8009e463e:36927 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T12:46:18,076 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(204): Hosts are {1ff8009e463e=0} racks are {/default-rack=0} 2024-11-25T12:46:18,076 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T12:46:18,076 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T12:46:18,076 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T12:46:18,076 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T12:46:18,076 INFO [1ff8009e463e:36927 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T12:46:18,076 INFO [1ff8009e463e:36927 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T12:46:18,076 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T12:46:18,077 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ff8009e463e,35205,1732538757028 2024-11-25T12:46:18,078 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ff8009e463e,35205,1732538757028, state=OPENING 2024-11-25T12:46:18,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:18,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:18,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:18,088 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:18,088 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:18,088 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:18,088 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:46:18,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ff8009e463e,35205,1732538757028}] 2024-11-25T12:46:18,241 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:46:18,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43897, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:46:18,247 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T12:46:18,247 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:18,247 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T12:46:18,249 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C35205%2C1732538757028.meta, suffix=.meta, logDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,35205,1732538757028, archiveDir=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs, maxLogs=32 2024-11-25T12:46:18,263 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,35205,1732538757028/1ff8009e463e%2C35205%2C1732538757028.meta.1732538778249.meta, exclude list is [], retry=0 2024-11-25T12:46:18,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:18,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:18,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:18,271 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,35205,1732538757028/1ff8009e463e%2C35205%2C1732538757028.meta.1732538778249.meta 2024-11-25T12:46:18,273 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:18,273 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:18,273 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T12:46:18,273 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T12:46:18,274 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T12:46:18,274 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T12:46:18,274 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:18,274 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T12:46:18,274 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T12:46:18,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:46:18,280 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:46:18,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:18,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:46:18,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:46:18,282 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:46:18,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:18,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:46:18,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:46:18,285 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:46:18,286 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:18,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:46:18,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:46:18,289 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:46:18,289 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:18,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:46:18,289 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:46:18,291 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740 2024-11-25T12:46:18,292 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740 2024-11-25T12:46:18,294 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-25T12:46:18,297 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:18,300 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-25T12:46:18,301 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-25T12:46:18,330 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/info/a8ccb7e820f64f4ea1bf409e504ab23b is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c./info:regioninfo/1732538777294/Put/seqid=0 2024-11-25T12:46:18,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741894_1072 (size=11177) 2024-11-25T12:46:18,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741894_1072 (size=11177) 2024-11-25T12:46:18,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741894_1072 (size=11177) 2024-11-25T12:46:18,348 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/info/a8ccb7e820f64f4ea1bf409e504ab23b 2024-11-25T12:46:18,389 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/ns/ab491160f94b4639bced73a016b06671 is 43, key is default/ns:d/1732538760695/Put/seqid=0 2024-11-25T12:46:18,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741895_1073 (size=5153) 2024-11-25T12:46:18,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741895_1073 (size=5153) 2024-11-25T12:46:18,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741895_1073 (size=5153) 2024-11-25T12:46:18,408 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/ns/ab491160f94b4639bced73a016b06671 2024-11-25T12:46:18,438 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/table/ba23fdf0af4349528aec87fbac8b5527 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732538775121/Put/seqid=0 2024-11-25T12:46:18,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741896_1074 (size=5431) 2024-11-25T12:46:18,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741896_1074 (size=5431) 2024-11-25T12:46:18,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741896_1074 (size=5431) 2024-11-25T12:46:18,456 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/table/ba23fdf0af4349528aec87fbac8b5527 2024-11-25T12:46:18,471 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/info/a8ccb7e820f64f4ea1bf409e504ab23b as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/info/a8ccb7e820f64f4ea1bf409e504ab23b 2024-11-25T12:46:18,483 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/info/a8ccb7e820f64f4ea1bf409e504ab23b, entries=36, sequenceid=18, filesize=10.9 K 2024-11-25T12:46:18,485 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/ns/ab491160f94b4639bced73a016b06671 as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/ns/ab491160f94b4639bced73a016b06671 2024-11-25T12:46:18,493 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/ns/ab491160f94b4639bced73a016b06671, entries=2, sequenceid=18, filesize=5.0 K 2024-11-25T12:46:18,494 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/table/ba23fdf0af4349528aec87fbac8b5527 as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/table/ba23fdf0af4349528aec87fbac8b5527 2024-11-25T12:46:18,500 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/table/ba23fdf0af4349528aec87fbac8b5527, entries=2, sequenceid=18, filesize=5.3 K 2024-11-25T12:46:18,500 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 199ms, sequenceid=18, compaction requested=false; wal=null 2024-11-25T12:46:18,500 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T12:46:18,501 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-25T12:46:18,503 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:46:18,503 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:46:18,503 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:46:18,505 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:46:18,507 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-25T12:46:18,508 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60669620, jitterRate=-0.09595221281051636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:46:18,508 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T12:46:18,509 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732538778274Writing region info on filesystem at 1732538778274Initializing all the Stores at 1732538778276 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538778276Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538778279 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538778279Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538778279Obtaining lock to block concurrent updates at 1732538778301 (+22 ms)Preparing flush snapshotting stores in 1588230740 at 1732538778301Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732538778301Flushing stores of hbase:meta,,1.1588230740 at 1732538778301Flushing 1588230740/info: creating writer at 1732538778301Flushing 1588230740/info: appending metadata at 1732538778330 (+29 ms)Flushing 1588230740/info: closing flushed file at 1732538778330Flushing 1588230740/ns: creating writer at 1732538778367 (+37 ms)Flushing 1588230740/ns: appending metadata at 1732538778388 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732538778388Flushing 1588230740/table: creating writer at 1732538778421 (+33 ms)Flushing 1588230740/table: appending metadata at 1732538778437 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732538778437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71867b36: reopening flushed file at 1732538778470 (+33 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57a4b67f: reopening flushed file at 1732538778484 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d4d5d53: reopening flushed file at 1732538778493 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 199ms, sequenceid=18, compaction requested=false; wal=null at 1732538778500 (+7 ms)Cleaning up temporary data from old regions at 1732538778503 (+3 ms)Running coprocessor post-open hooks at 1732538778508 (+5 ms)Region opened successfully at 1732538778509 (+1 ms) 2024-11-25T12:46:18,510 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732538778241 2024-11-25T12:46:18,512 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T12:46:18,512 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T12:46:18,513 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=1ff8009e463e,35205,1732538757028 2024-11-25T12:46:18,514 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ff8009e463e,35205,1732538757028, state=OPEN 2024-11-25T12:46:18,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:18,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:18,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:46:18,521 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ff8009e463e,35205,1732538757028 2024-11-25T12:46:18,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:18,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:18,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:46:18,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-25T12:46:18,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ff8009e463e,35205,1732538757028 in 433 msec 2024-11-25T12:46:18,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-25T12:46:18,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 605 msec 2024-11-25T12:46:18,526 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 1ff8009e463e,38825,1732538756840 had 2 regions 2024-11-25T12:46:18,527 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1ff8009e463e,38825,1732538756840, splitWal=true, meta=true, isMeta: false 2024-11-25T12:46:18,529 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1ff8009e463e,38825,1732538756840 WAL count=1, meta=false 2024-11-25T12:46:18,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1ff8009e463e%2C38825%2C1732538756840.1732538759929}] 2024-11-25T12:46:18,531 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:18,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1ff8009e463e%2C38825%2C1732538756840.1732538759929, worker=1ff8009e463e,43295,1732538756657}] 2024-11-25T12:46:18,542 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:46:18,543 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35205,1732538757028, seqNum=-1] 2024-11-25T12:46:18,543 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:46:18,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:46:18,662 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ff8009e463e:38825 this server is in the failed servers list 2024-11-25T12:46:18,663 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ff8009e463e:38825 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ff8009e463e:38825 2024-11-25T12:46:18,663 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ff8009e463e:38825 2024-11-25T12:46:18,663 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=12 from cache 2024-11-25T12:46:18,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43295 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-25T12:46:18,702 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929, size=1.4 K (1406bytes) 2024-11-25T12:46:18,702 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929 2024-11-25T12:46:18,702 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929 after 0ms 2024-11-25T12:46:18,705 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:18,705 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929 took 4ms 2024-11-25T12:46:18,708 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for c6d3a9a4a6e575e3ff59df3723af337c: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-25T12:46:18,708 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929 so closing down 2024-11-25T12:46:18,708 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:18,709 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:18,709 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 4 ms; skipped=6; WAL=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929, size=1.4 K, length=1406, corrupted=false, cancelled=false 2024-11-25T12:46:18,709 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929, journal: Splitting hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929, size=1.4 K (1406bytes) at 1732538778702Finishing writing output for hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929 so closing down at 1732538778708 (+6 ms)3 split writer threads finished at 1732538778709 (+1 ms)Processed 6 edits across 0 Regions in 4 ms; skipped=6; WAL=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929, size=1.4 K, length=1406, corrupted=false, cancelled=false at 1732538778709 2024-11-25T12:46:18,709 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929 2024-11-25T12:46:18,709 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-25T12:46:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36927 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-25T12:46:18,714 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting/1ff8009e463e%2C38825%2C1732538756840.1732538759929 to hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs 2024-11-25T12:46:18,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-25T12:46:18,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1ff8009e463e%2C38825%2C1732538756840.1732538759929, worker=1ff8009e463e,43295,1732538756657 in 182 msec 2024-11-25T12:46:18,718 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:18,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-25T12:46:18,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 1ff8009e463e%2C38825%2C1732538756840.1732538759929, worker=1ff8009e463e,43295,1732538756657 in 189 msec 2024-11-25T12:46:18,722 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,38825,1732538756840-splitting dir is empty, no logs to split. 2024-11-25T12:46:18,722 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1ff8009e463e,38825,1732538756840 WAL count=0, meta=false 2024-11-25T12:46:18,722 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1ff8009e463e,38825,1732538756840 WAL splitting is done? wals=0, meta=false 2024-11-25T12:46:18,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN}] 2024-11-25T12:46:18,726 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN 2024-11-25T12:46:18,727 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T12:46:18,877 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(204): Hosts are {1ff8009e463e=0} racks are {/default-rack=0} 2024-11-25T12:46:18,878 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T12:46:18,878 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T12:46:18,878 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T12:46:18,878 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T12:46:18,878 INFO [1ff8009e463e:36927 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T12:46:18,878 INFO [1ff8009e463e:36927 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T12:46:18,878 DEBUG [1ff8009e463e:36927 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T12:46:18,878 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=OPENING, regionLocation=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:18,880 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1ff8009e463e:38825 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ff8009e463e/172.17.0.3:38825 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:18,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1, error=java.net.ConnectException: Call to address=1ff8009e463e:38825 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ff8009e463e/172.17.0.3:38825 2024-11-25T12:46:18,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-25T12:46:18,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,38825,1732538756840, seqNum=-1 from cache 2024-11-25T12:46:18,880 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=18] 2024-11-25T12:46:18,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 1ff8009e463e:38825 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ff8009e463e/172.17.0.3:38825 2024-11-25T12:46:18,881 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ff8009e463e:38825 this server is in the failed servers list 2024-11-25T12:46:18,881 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ff8009e463e:38825 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ff8009e463e:38825 2024-11-25T12:46:18,881 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ff8009e463e:38825 2024-11-25T12:46:18,881 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,38825,1732538756840, seqNum=18 from cache 2024-11-25T12:46:18,982 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:46:18,982 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35205,1732538757028, seqNum=-1] 2024-11-25T12:46:18,982 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:46:18,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46781, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:46:18,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN because future has completed 2024-11-25T12:46:18,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657}] 2024-11-25T12:46:19,142 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:19,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => c6d3a9a4a6e575e3ff59df3723af337c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:19,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:19,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,145 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,147 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf1 2024-11-25T12:46:19,147 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:19,154 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf1/969e437bfc3a4ed2996bc9f6d1618a79 2024-11-25T12:46:19,155 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:19,155 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,156 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d3a9a4a6e575e3ff59df3723af337c columnFamilyName cf2 2024-11-25T12:46:19,156 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:19,164 DEBUG [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/cf2/b3f4dff6c2784ffbbe1977a0aee9fd91 2024-11-25T12:46:19,164 INFO [StoreOpener-c6d3a9a4a6e575e3ff59df3723af337c-1 {}] regionserver.HStore(327): Store=c6d3a9a4a6e575e3ff59df3723af337c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:19,164 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,165 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,167 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,168 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,168 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,168 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T12:46:19,171 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,173 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened c6d3a9a4a6e575e3ff59df3723af337c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63168403, jitterRate=-0.05871744453907013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T12:46:19,173 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:19,174 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for c6d3a9a4a6e575e3ff59df3723af337c: Running coprocessor pre-open hook at 1732538779144Writing region info on filesystem at 1732538779144Initializing all the Stores at 1732538779145 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538779145Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538779145Cleaning up temporary data from old regions at 1732538779168 (+23 ms)Running coprocessor post-open hooks at 1732538779173 (+5 ms)Region opened successfully at 1732538779174 (+1 ms) 2024-11-25T12:46:19,175 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., pid=21, masterSystemTime=1732538779138 2024-11-25T12:46:19,179 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:19,179 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:19,181 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c6d3a9a4a6e575e3ff59df3723af337c, regionState=OPEN, openSeqNum=18, regionLocation=1ff8009e463e,43295,1732538756657 2024-11-25T12:46:19,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657 because future has completed 2024-11-25T12:46:19,189 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36927 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1ff8009e463e,43295,1732538756657, table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T12:46:19,198 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c., hostname=1ff8009e463e,43295,1732538756657, seqNum=18] 2024-11-25T12:46:19,198 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-25T12:46:19,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure c6d3a9a4a6e575e3ff59df3723af337c, server=1ff8009e463e,43295,1732538756657 in 205 msec 2024-11-25T12:46:19,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-25T12:46:19,207 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 1ff8009e463e,38825,1732538756840 after splitting done 2024-11-25T12:46:19,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c6d3a9a4a6e575e3ff59df3723af337c, ASSIGN in 475 msec 2024-11-25T12:46:19,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 1ff8009e463e,38825,1732538756840, splitWal=true, meta=true in 1.5510 sec 2024-11-25T12:46:19,231 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=406 (was 410), OpenFileDescriptor=1034 (was 1005) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=639 (was 660), ProcessCount=11 (was 11), AvailableMemoryMB=1993 (was 1328) - AvailableMemoryMB LEAK? - 2024-11-25T12:46:19,233 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1034 is superior to 1024 2024-11-25T12:46:19,260 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1034, MaxFileDescriptor=1048576, SystemLoadAverage=639, ProcessCount=11, AvailableMemoryMB=1988 2024-11-25T12:46:19,260 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1034 is superior to 1024 2024-11-25T12:46:19,307 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:19,309 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:19,312 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:19,315 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33088301, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-33088301, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:19,333 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33088301/hregion-33088301.1732538779316, exclude list is [], retry=0 2024-11-25T12:46:19,343 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:19,347 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:19,352 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:19,356 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33088301/hregion-33088301.1732538779316 2024-11-25T12:46:19,359 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:19,360 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c2146912412e1af293cfd04a4dd9c7d9, NAME => 'testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:19,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741898_1076 (size=67) 2024-11-25T12:46:19,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741898_1076 (size=67) 2024-11-25T12:46:19,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741898_1076 (size=67) 2024-11-25T12:46:19,800 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:19,802 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,804 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName a 2024-11-25T12:46:19,804 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:19,805 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:19,805 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,807 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName b 2024-11-25T12:46:19,808 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:19,808 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:19,808 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,810 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName c 2024-11-25T12:46:19,810 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:19,811 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:19,811 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,812 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,813 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,815 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,815 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,815 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:19,817 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,819 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:19,820 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c2146912412e1af293cfd04a4dd9c7d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68982558, jitterRate=0.02792021632194519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:19,820 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c2146912412e1af293cfd04a4dd9c7d9: Writing region info on filesystem at 1732538779800Initializing all the Stores at 1732538779801 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538779801Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538779802 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538779802Cleaning up temporary data from old regions at 1732538779815 (+13 ms)Region opened successfully at 1732538779820 (+5 ms) 2024-11-25T12:46:19,820 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c2146912412e1af293cfd04a4dd9c7d9, disabling compactions & flushes 2024-11-25T12:46:19,821 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. 2024-11-25T12:46:19,821 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. 2024-11-25T12:46:19,821 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. after waiting 0 ms 2024-11-25T12:46:19,821 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. 2024-11-25T12:46:19,821 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. 2024-11-25T12:46:19,821 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c2146912412e1af293cfd04a4dd9c7d9: Waiting for close lock at 1732538779820Disabling compacts and flushes for region at 1732538779820Disabling writes for close at 1732538779821 (+1 ms)Writing region close event to WAL at 1732538779821Closed at 1732538779821 2024-11-25T12:46:19,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741897_1075 (size=95) 2024-11-25T12:46:19,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741897_1075 (size=95) 2024-11-25T12:46:19,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741897_1075 (size=95) 2024-11-25T12:46:19,826 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:19,826 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33088301:(num 1732538779316) 2024-11-25T12:46:19,826 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:19,828 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:19,840 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828, exclude list is [], retry=0 2024-11-25T12:46:19,843 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:19,843 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:19,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:19,845 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828 2024-11-25T12:46:19,846 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:19,846 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c2146912412e1af293cfd04a4dd9c7d9, NAME => 'testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:19,846 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:19,846 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,846 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,848 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,849 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName a 2024-11-25T12:46:19,849 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:19,849 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:19,850 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,850 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName b 2024-11-25T12:46:19,850 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:19,851 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:19,851 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,852 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName c 2024-11-25T12:46:19,852 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:19,853 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:19,853 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,854 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,855 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,855 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,856 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:19,857 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:19,858 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c2146912412e1af293cfd04a4dd9c7d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66805415, jitterRate=-0.0045217424631118774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:19,858 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c2146912412e1af293cfd04a4dd9c7d9: Writing region info on filesystem at 1732538779846Initializing all the Stores at 1732538779847 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538779847Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538779847Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538779847Cleaning up temporary data from old regions at 1732538779855 (+8 ms)Region opened successfully at 1732538779858 (+3 ms) 2024-11-25T12:46:19,892 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c2146912412e1af293cfd04a4dd9c7d9 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-25T12:46:19,919 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/a/dc0191d0201848db8260cb6648b91ce8 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732538779858/Put/seqid=0 2024-11-25T12:46:19,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741900_1078 (size=5958) 2024-11-25T12:46:19,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741900_1078 (size=5958) 2024-11-25T12:46:19,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741900_1078 (size=5958) 2024-11-25T12:46:19,977 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/a/dc0191d0201848db8260cb6648b91ce8 2024-11-25T12:46:20,033 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/b/0aa736a6a8c74099acaed0ad96649087 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732538779870/Put/seqid=0 2024-11-25T12:46:20,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741901_1079 (size=5958) 2024-11-25T12:46:20,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741901_1079 (size=5958) 2024-11-25T12:46:20,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741901_1079 (size=5958) 2024-11-25T12:46:20,058 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/b/0aa736a6a8c74099acaed0ad96649087 2024-11-25T12:46:20,082 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/c/e7dc2271270e487b81b6bb5e1949ff26 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732538779881/Put/seqid=0 2024-11-25T12:46:20,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741902_1080 (size=5958) 2024-11-25T12:46:20,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741902_1080 (size=5958) 2024-11-25T12:46:20,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741902_1080 (size=5958) 2024-11-25T12:46:20,097 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/c/e7dc2271270e487b81b6bb5e1949ff26 2024-11-25T12:46:20,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/a/dc0191d0201848db8260cb6648b91ce8 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/a/dc0191d0201848db8260cb6648b91ce8 2024-11-25T12:46:20,115 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/a/dc0191d0201848db8260cb6648b91ce8, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T12:46:20,117 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/b/0aa736a6a8c74099acaed0ad96649087 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/b/0aa736a6a8c74099acaed0ad96649087 2024-11-25T12:46:20,127 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/b/0aa736a6a8c74099acaed0ad96649087, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T12:46:20,130 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/c/e7dc2271270e487b81b6bb5e1949ff26 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/c/e7dc2271270e487b81b6bb5e1949ff26 2024-11-25T12:46:20,138 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/c/e7dc2271270e487b81b6bb5e1949ff26, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T12:46:20,139 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c2146912412e1af293cfd04a4dd9c7d9 in 247ms, sequenceid=33, compaction requested=false 2024-11-25T12:46:20,140 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c2146912412e1af293cfd04a4dd9c7d9: 2024-11-25T12:46:20,140 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c2146912412e1af293cfd04a4dd9c7d9, disabling compactions & flushes 2024-11-25T12:46:20,140 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. 2024-11-25T12:46:20,140 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. 2024-11-25T12:46:20,140 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. after waiting 0 ms 2024-11-25T12:46:20,140 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. 2024-11-25T12:46:20,152 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. 2024-11-25T12:46:20,152 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c2146912412e1af293cfd04a4dd9c7d9: Waiting for close lock at 1732538780140Disabling compacts and flushes for region at 1732538780140Disabling writes for close at 1732538780140Writing region close event to WAL at 1732538780151 (+11 ms)Closed at 1732538780151 2024-11-25T12:46:20,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741899_1077 (size=3383) 2024-11-25T12:46:20,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741899_1077 (size=3383) 2024-11-25T12:46:20,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741899_1077 (size=3383) 2024-11-25T12:46:20,159 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/b/0aa736a6a8c74099acaed0ad96649087 to hdfs://localhost:35793/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/b/0aa736a6a8c74099acaed0ad96649087 2024-11-25T12:46:20,179 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828, size=3.3 K (3383bytes) 2024-11-25T12:46:20,179 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828 2024-11-25T12:46:20,179 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828 after 0ms 2024-11-25T12:46:20,182 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:20,182 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828 took 4ms 2024-11-25T12:46:20,185 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828 so closing down 2024-11-25T12:46:20,186 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:20,189 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538779828.temp 2024-11-25T12:46:20,190 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000003-wal.1732538779828.temp 2024-11-25T12:46:20,191 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741903_1081 (size=2944) 2024-11-25T12:46:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741903_1081 (size=2944) 2024-11-25T12:46:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741903_1081 (size=2944) 2024-11-25T12:46:20,617 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:46:20,627 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000003-wal.1732538779828.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:20,629 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000003-wal.1732538779828.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000032 2024-11-25T12:46:20,630 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 447 ms; skipped=2; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828, size=3.3 K, length=3383, corrupted=false, cancelled=false 2024-11-25T12:46:20,630 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828, journal: Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828, size=3.3 K (3383bytes) at 1732538780179Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828 so closing down at 1732538780185 (+6 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000003-wal.1732538779828.temp at 1732538780191 (+6 ms)3 split writer threads finished at 1732538780191Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000003-wal.1732538779828.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732538780627 (+436 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000003-wal.1732538779828.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000032 at 1732538780630 (+3 ms)Processed 32 edits across 1 Regions in 447 ms; skipped=2; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828, size=3.3 K, length=3383, corrupted=false, cancelled=false at 1732538780630 2024-11-25T12:46:20,633 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538779828 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538779828 2024-11-25T12:46:20,635 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000032 2024-11-25T12:46:20,635 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:20,638 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:20,658 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538780638, exclude list is [], retry=0 2024-11-25T12:46:20,661 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:20,661 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:20,662 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:20,672 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538780638 2024-11-25T12:46:20,672 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:20,672 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c2146912412e1af293cfd04a4dd9c7d9, NAME => 'testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:20,672 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:20,672 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,672 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,674 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,676 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName a 2024-11-25T12:46:20,676 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:20,684 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/a/dc0191d0201848db8260cb6648b91ce8 2024-11-25T12:46:20,685 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:20,685 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,687 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName b 2024-11-25T12:46:20,687 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:20,688 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:20,688 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,689 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2146912412e1af293cfd04a4dd9c7d9 columnFamilyName c 2024-11-25T12:46:20,689 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:20,696 DEBUG [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/c/e7dc2271270e487b81b6bb5e1949ff26 2024-11-25T12:46:20,696 INFO [StoreOpener-c2146912412e1af293cfd04a4dd9c7d9-1 {}] regionserver.HStore(327): Store=c2146912412e1af293cfd04a4dd9c7d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:20,696 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,697 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,698 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,699 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000032 2024-11-25T12:46:20,701 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:20,703 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000032 2024-11-25T12:46:20,703 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c2146912412e1af293cfd04a4dd9c7d9 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-25T12:46:20,719 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/b/702eb311e052492fab4a276bb5b1d360 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732538779870/Put/seqid=0 2024-11-25T12:46:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741905_1083 (size=5958) 2024-11-25T12:46:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741905_1083 (size=5958) 2024-11-25T12:46:20,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741905_1083 (size=5958) 2024-11-25T12:46:20,734 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/b/702eb311e052492fab4a276bb5b1d360 2024-11-25T12:46:20,741 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/.tmp/b/702eb311e052492fab4a276bb5b1d360 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/b/702eb311e052492fab4a276bb5b1d360 2024-11-25T12:46:20,747 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/b/702eb311e052492fab4a276bb5b1d360, entries=10, sequenceid=32, filesize=5.8 K 2024-11-25T12:46:20,747 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c2146912412e1af293cfd04a4dd9c7d9 in 44ms, sequenceid=32, compaction requested=false; wal=null 2024-11-25T12:46:20,748 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/0000000000000000032 2024-11-25T12:46:20,752 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,752 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,753 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:20,754 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c2146912412e1af293cfd04a4dd9c7d9 2024-11-25T12:46:20,757 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/c2146912412e1af293cfd04a4dd9c7d9/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-25T12:46:20,758 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c2146912412e1af293cfd04a4dd9c7d9; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66787779, jitterRate=-0.004784539341926575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:20,758 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c2146912412e1af293cfd04a4dd9c7d9: Writing region info on filesystem at 1732538780673Initializing all the Stores at 1732538780674 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538780674Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538780674Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538780674Obtaining lock to block concurrent updates at 1732538780703 (+29 ms)Preparing flush snapshotting stores in c2146912412e1af293cfd04a4dd9c7d9 at 1732538780703Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732538780703Flushing stores of testReplayEditsWrittenViaHRegion,,1732538779308.c2146912412e1af293cfd04a4dd9c7d9. at 1732538780703Flushing c2146912412e1af293cfd04a4dd9c7d9/b: creating writer at 1732538780704 (+1 ms)Flushing c2146912412e1af293cfd04a4dd9c7d9/b: appending metadata at 1732538780718 (+14 ms)Flushing c2146912412e1af293cfd04a4dd9c7d9/b: closing flushed file at 1732538780718Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cd1dcb0: reopening flushed file at 1732538780740 (+22 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c2146912412e1af293cfd04a4dd9c7d9 in 44ms, sequenceid=32, compaction requested=false; wal=null at 1732538780747 (+7 ms)Cleaning up temporary data from old regions at 1732538780752 (+5 ms)Region opened successfully at 1732538780758 (+6 ms) 2024-11-25T12:46:20,781 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=413 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:40806 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:33132 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:49842 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:49642 [Waiting for operation #52] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:33328 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:40600 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1104 (was 1034) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=639 (was 639), ProcessCount=11 (was 11), AvailableMemoryMB=1943 (was 1988) 2024-11-25T12:46:20,781 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1104 is superior to 1024 2024-11-25T12:46:20,792 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=413, OpenFileDescriptor=1104, MaxFileDescriptor=1048576, SystemLoadAverage=639, ProcessCount=11, AvailableMemoryMB=1942 2024-11-25T12:46:20,792 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1104 is superior to 1024 2024-11-25T12:46:20,807 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:20,810 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:20,810 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:20,813 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-06514189, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-06514189, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:20,827 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-06514189/hregion-06514189.1732538780813, exclude list is [], retry=0 2024-11-25T12:46:20,830 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:20,830 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:20,831 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:20,843 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-06514189/hregion-06514189.1732538780813 2024-11-25T12:46:20,843 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:20,844 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c70a7c6f92c33b1abca56ca3ab6dde45, NAME => 'testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:20,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741907_1085 (size=68) 2024-11-25T12:46:20,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741907_1085 (size=68) 2024-11-25T12:46:20,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741907_1085 (size=68) 2024-11-25T12:46:20,878 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:20,879 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,881 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName a 2024-11-25T12:46:20,881 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:20,882 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:20,882 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,885 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName b 2024-11-25T12:46:20,886 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:20,892 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:20,892 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,894 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName c 2024-11-25T12:46:20,894 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:20,897 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:20,897 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,908 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,909 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,911 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,911 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,912 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:20,915 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:20,934 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:20,935 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c70a7c6f92c33b1abca56ca3ab6dde45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68277640, jitterRate=0.01741611957550049}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:20,936 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c70a7c6f92c33b1abca56ca3ab6dde45: Writing region info on filesystem at 1732538780878Initializing all the Stores at 1732538780879 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538780879Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538780879Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538780879Cleaning up temporary data from old regions at 1732538780911 (+32 ms)Region opened successfully at 1732538780936 (+25 ms) 2024-11-25T12:46:20,936 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c70a7c6f92c33b1abca56ca3ab6dde45, disabling compactions & flushes 2024-11-25T12:46:20,936 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:20,936 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:20,936 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. after waiting 0 ms 2024-11-25T12:46:20,936 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:20,937 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:20,937 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c70a7c6f92c33b1abca56ca3ab6dde45: Waiting for close lock at 1732538780936Disabling compacts and flushes for region at 1732538780936Disabling writes for close at 1732538780936Writing region close event to WAL at 1732538780937 (+1 ms)Closed at 1732538780937 2024-11-25T12:46:20,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741906_1084 (size=95) 2024-11-25T12:46:20,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741906_1084 (size=95) 2024-11-25T12:46:20,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741906_1084 (size=95) 2024-11-25T12:46:20,951 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:20,951 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-06514189:(num 1732538780813) 2024-11-25T12:46:20,952 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:20,955 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:20,973 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955, exclude list is [], retry=0 2024-11-25T12:46:20,976 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:20,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:20,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:21,000 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955 2024-11-25T12:46:21,003 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:21,152 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c70a7c6f92c33b1abca56ca3ab6dde45, NAME => 'testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:21,157 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,157 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:21,158 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,158 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,164 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,166 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName a 2024-11-25T12:46:21,166 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:21,167 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:21,167 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,171 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName b 2024-11-25T12:46:21,171 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:21,172 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:21,172 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,174 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName c 2024-11-25T12:46:21,174 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:21,174 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:21,175 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,179 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,184 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,185 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,185 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:21,189 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,190 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c70a7c6f92c33b1abca56ca3ab6dde45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58858205, jitterRate=-0.12294439971446991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:21,190 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:21,191 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c70a7c6f92c33b1abca56ca3ab6dde45: Running coprocessor pre-open hook at 1732538781158Writing region info on filesystem at 1732538781158Initializing all the Stores at 1732538781160 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538781160Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538781160Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538781160Cleaning up temporary data from old regions at 1732538781185 (+25 ms)Running coprocessor post-open hooks at 1732538781190 (+5 ms)Region opened successfully at 1732538781190 2024-11-25T12:46:21,215 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c70a7c6f92c33b1abca56ca3ab6dde45 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-25T12:46:21,216 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:21,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741835_1011 (size=1406) 2024-11-25T12:46:22,217 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:23,218 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:24,219 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:24,378 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:46:25,220 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:26,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-25T12:46:26,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:26,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-25T12:46:26,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:26,221 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:26,660 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:46:27,221 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:28,222 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:29,223 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:30,224 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c70a7c6f92c33b1abca56ca3ab6dde45/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:30,225 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c70a7c6f92c33b1abca56ca3ab6dde45: 2024-11-25T12:46:30,225 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:30,240 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c70a7c6f92c33b1abca56ca3ab6dde45: 2024-11-25T12:46:30,240 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-25T12:46:30,241 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c70a7c6f92c33b1abca56ca3ab6dde45, disabling compactions & flushes 2024-11-25T12:46:30,241 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:30,241 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:30,241 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. after waiting 0 ms 2024-11-25T12:46:30,241 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:30,241 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:30,242 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. 2024-11-25T12:46:30,242 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c70a7c6f92c33b1abca56ca3ab6dde45: Waiting for close lock at 1732538790241Running coprocessor pre-close hooks at 1732538790241Disabling compacts and flushes for region at 1732538790241Disabling writes for close at 1732538790241Writing region close event to WAL at 1732538790241Running coprocessor post-close hooks at 1732538790241Closed at 1732538790242 (+1 ms) 2024-11-25T12:46:30,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741908_1086 (size=2685) 2024-11-25T12:46:30,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741908_1086 (size=2685) 2024-11-25T12:46:30,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741908_1086 (size=2685) 2024-11-25T12:46:30,262 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955, size=2.6 K (2685bytes) 2024-11-25T12:46:30,262 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955 2024-11-25T12:46:30,263 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955 after 1ms 2024-11-25T12:46:30,265 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:30,265 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955 took 3ms 2024-11-25T12:46:30,267 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955 so closing down 2024-11-25T12:46:30,267 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:30,268 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732538780955.temp 2024-11-25T12:46:30,269 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000004-wal.1732538780955.temp 2024-11-25T12:46:30,270 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:30,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741909_1087 (size=2094) 2024-11-25T12:46:30,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741909_1087 (size=2094) 2024-11-25T12:46:30,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741909_1087 (size=2094) 2024-11-25T12:46:30,278 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000004-wal.1732538780955.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:30,279 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000004-wal.1732538780955.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000026 2024-11-25T12:46:30,279 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955, size=2.6 K, length=2685, corrupted=false, cancelled=false 2024-11-25T12:46:30,279 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955, journal: Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955, size=2.6 K (2685bytes) at 1732538790262Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955 so closing down at 1732538790267 (+5 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000004-wal.1732538780955.temp at 1732538790269 (+2 ms)3 split writer threads finished at 1732538790270 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000004-wal.1732538780955.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732538790278 (+8 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000004-wal.1732538780955.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000026 at 1732538790279 (+1 ms)Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955, size=2.6 K, length=2685, corrupted=false, cancelled=false at 1732538790279 2024-11-25T12:46:30,281 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538780955 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538780955 2024-11-25T12:46:30,282 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000026 2024-11-25T12:46:30,282 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:30,287 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:30,303 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538790287, exclude list is [], retry=0 2024-11-25T12:46:30,306 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:30,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:30,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:30,309 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538790287 2024-11-25T12:46:30,311 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:30,312 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c70a7c6f92c33b1abca56ca3ab6dde45, NAME => 'testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:30,312 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,313 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:30,313 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,313 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,316 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,317 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName a 2024-11-25T12:46:30,317 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:30,318 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:30,318 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,320 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName b 2024-11-25T12:46:30,320 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:30,320 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:30,321 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,322 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c70a7c6f92c33b1abca56ca3ab6dde45 columnFamilyName c 2024-11-25T12:46:30,322 DEBUG [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:30,322 INFO [StoreOpener-c70a7c6f92c33b1abca56ca3ab6dde45-1 {}] regionserver.HStore(327): Store=c70a7c6f92c33b1abca56ca3ab6dde45/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:30,323 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,323 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,325 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,326 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000026 2024-11-25T12:46:30,328 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:30,330 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000026 2024-11-25T12:46:30,330 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c70a7c6f92c33b1abca56ca3ab6dde45 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-25T12:46:30,351 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/a/303d004b5ac34c4599f2a322533f12a1 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732538790228/Put/seqid=0 2024-11-25T12:46:30,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741911_1089 (size=5523) 2024-11-25T12:46:30,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741911_1089 (size=5523) 2024-11-25T12:46:30,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741911_1089 (size=5523) 2024-11-25T12:46:30,367 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/a/303d004b5ac34c4599f2a322533f12a1 2024-11-25T12:46:30,389 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/b/2b8d3adb16864a3f9ba17676b4bdd326 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732538790225/Put/seqid=0 2024-11-25T12:46:30,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741912_1090 (size=5524) 2024-11-25T12:46:30,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741912_1090 (size=5524) 2024-11-25T12:46:30,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741912_1090 (size=5524) 2024-11-25T12:46:30,412 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/b/2b8d3adb16864a3f9ba17676b4bdd326 2024-11-25T12:46:30,433 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/c/f8322952886f4f578244e8f1d48bbacd is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732538790227/Put/seqid=0 2024-11-25T12:46:30,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741913_1091 (size=5457) 2024-11-25T12:46:30,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741913_1091 (size=5457) 2024-11-25T12:46:30,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741913_1091 (size=5457) 2024-11-25T12:46:30,442 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/c/f8322952886f4f578244e8f1d48bbacd 2024-11-25T12:46:30,450 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/a/303d004b5ac34c4599f2a322533f12a1 as hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/a/303d004b5ac34c4599f2a322533f12a1 2024-11-25T12:46:30,472 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/a/303d004b5ac34c4599f2a322533f12a1, entries=7, sequenceid=26, filesize=5.4 K 2024-11-25T12:46:30,489 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/b/2b8d3adb16864a3f9ba17676b4bdd326 as hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/b/2b8d3adb16864a3f9ba17676b4bdd326 2024-11-25T12:46:30,498 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/b/2b8d3adb16864a3f9ba17676b4bdd326, entries=7, sequenceid=26, filesize=5.4 K 2024-11-25T12:46:30,502 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/.tmp/c/f8322952886f4f578244e8f1d48bbacd as hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/c/f8322952886f4f578244e8f1d48bbacd 2024-11-25T12:46:30,518 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/c/f8322952886f4f578244e8f1d48bbacd, entries=6, sequenceid=26, filesize=5.3 K 2024-11-25T12:46:30,518 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for c70a7c6f92c33b1abca56ca3ab6dde45 in 188ms, sequenceid=26, compaction requested=false; wal=null 2024-11-25T12:46:30,519 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/0000000000000000026 2024-11-25T12:46:30,521 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,521 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,522 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:30,523 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,526 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsAfterAbortingFlush/c70a7c6f92c33b1abca56ca3ab6dde45/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-25T12:46:30,527 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c70a7c6f92c33b1abca56ca3ab6dde45; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62349483, jitterRate=-0.07092030346393585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:30,527 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c70a7c6f92c33b1abca56ca3ab6dde45 2024-11-25T12:46:30,528 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c70a7c6f92c33b1abca56ca3ab6dde45: Running coprocessor pre-open hook at 1732538790313Writing region info on filesystem at 1732538790313Initializing all the Stores at 1732538790314 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538790314Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538790315 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538790315Obtaining lock to block concurrent updates at 1732538790330 (+15 ms)Preparing flush snapshotting stores in c70a7c6f92c33b1abca56ca3ab6dde45 at 1732538790330Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732538790330Flushing stores of testReplayEditsAfterAbortingFlush,,1732538780808.c70a7c6f92c33b1abca56ca3ab6dde45. at 1732538790330Flushing c70a7c6f92c33b1abca56ca3ab6dde45/a: creating writer at 1732538790331 (+1 ms)Flushing c70a7c6f92c33b1abca56ca3ab6dde45/a: appending metadata at 1732538790350 (+19 ms)Flushing c70a7c6f92c33b1abca56ca3ab6dde45/a: closing flushed file at 1732538790350Flushing c70a7c6f92c33b1abca56ca3ab6dde45/b: creating writer at 1732538790373 (+23 ms)Flushing c70a7c6f92c33b1abca56ca3ab6dde45/b: appending metadata at 1732538790388 (+15 ms)Flushing c70a7c6f92c33b1abca56ca3ab6dde45/b: closing flushed file at 1732538790388Flushing c70a7c6f92c33b1abca56ca3ab6dde45/c: creating writer at 1732538790417 (+29 ms)Flushing c70a7c6f92c33b1abca56ca3ab6dde45/c: appending metadata at 1732538790432 (+15 ms)Flushing c70a7c6f92c33b1abca56ca3ab6dde45/c: closing flushed file at 1732538790432Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74322f15: reopening flushed file at 1732538790448 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73aed2f2: reopening flushed file at 1732538790472 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@689e9df5: reopening flushed file at 1732538790498 (+26 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for c70a7c6f92c33b1abca56ca3ab6dde45 in 188ms, sequenceid=26, compaction requested=false; wal=null at 1732538790518 (+20 ms)Cleaning up temporary data from old regions at 1732538790521 (+3 ms)Running coprocessor post-open hooks at 1732538790527 (+6 ms)Region opened successfully at 1732538790528 (+1 ms) 2024-11-25T12:46:30,551 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=417 (was 413) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:55190 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:55172 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:59470 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:60374 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:60400 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:59462 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1174 (was 1104) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=595 (was 639), ProcessCount=11 (was 11), AvailableMemoryMB=1881 (was 1942) 2024-11-25T12:46:30,551 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1174 is superior to 1024 2024-11-25T12:46:30,567 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1174, MaxFileDescriptor=1048576, SystemLoadAverage=595, ProcessCount=11, AvailableMemoryMB=1879 2024-11-25T12:46:30,567 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1174 is superior to 1024 2024-11-25T12:46:30,589 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:30,592 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:30,593 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:30,599 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37186144, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-37186144, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:30,621 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37186144/hregion-37186144.1732538790600, exclude list is [], retry=0 2024-11-25T12:46:30,636 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:30,641 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:30,644 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:30,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37186144/hregion-37186144.1732538790600 2024-11-25T12:46:30,659 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:30,660 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7e66b7e2cad77d6ac6ffe4f9835a87a7, NAME => 'testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:30,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741915_1093 (size=61) 2024-11-25T12:46:30,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741915_1093 (size=61) 2024-11-25T12:46:30,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741915_1093 (size=61) 2024-11-25T12:46:30,678 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:30,680 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,681 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e66b7e2cad77d6ac6ffe4f9835a87a7 columnFamilyName a 2024-11-25T12:46:30,681 DEBUG [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:30,682 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.HStore(327): Store=7e66b7e2cad77d6ac6ffe4f9835a87a7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:30,682 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,682 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,683 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,683 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,683 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,685 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,686 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:30,687 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7e66b7e2cad77d6ac6ffe4f9835a87a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66404445, jitterRate=-0.010496661067008972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:30,688 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7e66b7e2cad77d6ac6ffe4f9835a87a7: Writing region info on filesystem at 1732538790679Initializing all the Stores at 1732538790679Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538790679Cleaning up temporary data from old regions at 1732538790683 (+4 ms)Region opened successfully at 1732538790688 (+5 ms) 2024-11-25T12:46:30,688 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7e66b7e2cad77d6ac6ffe4f9835a87a7, disabling compactions & flushes 2024-11-25T12:46:30,688 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,688 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,688 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. after waiting 0 ms 2024-11-25T12:46:30,688 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,688 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,688 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7e66b7e2cad77d6ac6ffe4f9835a87a7: Waiting for close lock at 1732538790688Disabling compacts and flushes for region at 1732538790688Disabling writes for close at 1732538790688Writing region close event to WAL at 1732538790688Closed at 1732538790688 2024-11-25T12:46:30,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741914_1092 (size=95) 2024-11-25T12:46:30,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741914_1092 (size=95) 2024-11-25T12:46:30,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741914_1092 (size=95) 2024-11-25T12:46:30,692 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:30,692 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37186144:(num 1732538790600) 2024-11-25T12:46:30,693 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:30,695 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:30,710 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695, exclude list is [], retry=0 2024-11-25T12:46:30,713 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:30,713 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:30,713 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:30,715 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695 2024-11-25T12:46:30,715 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:30,715 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e66b7e2cad77d6ac6ffe4f9835a87a7, NAME => 'testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:30,715 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:30,715 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,715 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,718 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,719 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e66b7e2cad77d6ac6ffe4f9835a87a7 columnFamilyName a 2024-11-25T12:46:30,719 DEBUG [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:30,720 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.HStore(327): Store=7e66b7e2cad77d6ac6ffe4f9835a87a7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:30,720 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,721 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,722 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,722 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,722 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,724 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:30,724 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7e66b7e2cad77d6ac6ffe4f9835a87a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66428461, jitterRate=-0.010138794779777527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:30,725 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7e66b7e2cad77d6ac6ffe4f9835a87a7: Writing region info on filesystem at 1732538790715Initializing all the Stores at 1732538790716 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538790716Cleaning up temporary data from old regions at 1732538790722 (+6 ms)Region opened successfully at 1732538790725 (+3 ms) 2024-11-25T12:46:30,733 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7e66b7e2cad77d6ac6ffe4f9835a87a7, disabling compactions & flushes 2024-11-25T12:46:30,733 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,733 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,733 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. after waiting 0 ms 2024-11-25T12:46:30,733 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,733 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,733 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. 2024-11-25T12:46:30,734 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7e66b7e2cad77d6ac6ffe4f9835a87a7: Waiting for close lock at 1732538790733Disabling compacts and flushes for region at 1732538790733Disabling writes for close at 1732538790733Writing region close event to WAL at 1732538790733Closed at 1732538790733 2024-11-25T12:46:30,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741916_1094 (size=1050) 2024-11-25T12:46:30,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741916_1094 (size=1050) 2024-11-25T12:46:30,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741916_1094 (size=1050) 2024-11-25T12:46:30,753 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695, size=1.0 K (1050bytes) 2024-11-25T12:46:30,753 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695 2024-11-25T12:46:30,754 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695 after 1ms 2024-11-25T12:46:30,757 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:30,757 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695 took 4ms 2024-11-25T12:46:30,759 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695 so closing down 2024-11-25T12:46:30,759 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:30,760 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538790695.temp 2024-11-25T12:46:30,761 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000003-wal.1732538790695.temp 2024-11-25T12:46:30,762 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:30,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741917_1095 (size=1050) 2024-11-25T12:46:30,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741917_1095 (size=1050) 2024-11-25T12:46:30,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741917_1095 (size=1050) 2024-11-25T12:46:30,768 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000003-wal.1732538790695.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:30,769 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000003-wal.1732538790695.temp to hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000012 2024-11-25T12:46:30,770 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-25T12:46:30,770 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695, journal: Splitting hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695, size=1.0 K (1050bytes) at 1732538790753Finishing writing output for hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695 so closing down at 1732538790759 (+6 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000003-wal.1732538790695.temp at 1732538790762 (+3 ms)3 split writer threads finished at 1732538790762Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000003-wal.1732538790695.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732538790768 (+6 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000003-wal.1732538790695.temp to hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000012 at 1732538790769 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1732538790770 (+1 ms) 2024-11-25T12:46:30,771 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538790695 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538790695 2024-11-25T12:46:30,772 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000012 2024-11-25T12:46:30,775 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:31,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:31,161 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:31,179 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538791161, exclude list is [], retry=0 2024-11-25T12:46:31,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:31,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:31,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:31,187 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538791161 2024-11-25T12:46:31,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:31,188 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e66b7e2cad77d6ac6ffe4f9835a87a7, NAME => 'testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:31,188 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:31,188 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,188 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,193 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,195 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e66b7e2cad77d6ac6ffe4f9835a87a7 columnFamilyName a 2024-11-25T12:46:31,195 DEBUG [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:31,196 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.HStore(327): Store=7e66b7e2cad77d6ac6ffe4f9835a87a7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:31,196 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,197 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,199 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,200 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000012 2024-11-25T12:46:31,202 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:31,204 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000012 2024-11-25T12:46:31,204 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7e66b7e2cad77d6ac6ffe4f9835a87a7 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-25T12:46:31,230 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/.tmp/a/aac1c1ec277e4917a090c4de5e8a5b8e is 79, key is testDatalossWhenInputError/a:x0/1732538790725/Put/seqid=0 2024-11-25T12:46:31,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741919_1097 (size=5808) 2024-11-25T12:46:31,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741919_1097 (size=5808) 2024-11-25T12:46:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741919_1097 (size=5808) 2024-11-25T12:46:31,241 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/.tmp/a/aac1c1ec277e4917a090c4de5e8a5b8e 2024-11-25T12:46:31,252 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/.tmp/a/aac1c1ec277e4917a090c4de5e8a5b8e as hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/a/aac1c1ec277e4917a090c4de5e8a5b8e 2024-11-25T12:46:31,261 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/a/aac1c1ec277e4917a090c4de5e8a5b8e, entries=10, sequenceid=12, filesize=5.7 K 2024-11-25T12:46:31,261 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7e66b7e2cad77d6ac6ffe4f9835a87a7 in 57ms, sequenceid=12, compaction requested=false; wal=null 2024-11-25T12:46:31,262 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/0000000000000000012 2024-11-25T12:46:31,263 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,263 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,267 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,269 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-25T12:46:31,270 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7e66b7e2cad77d6ac6ffe4f9835a87a7; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64415524, jitterRate=-0.040133893489837646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:31,271 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7e66b7e2cad77d6ac6ffe4f9835a87a7: Writing region info on filesystem at 1732538791188Initializing all the Stores at 1732538791192 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538791193 (+1 ms)Obtaining lock to block concurrent updates at 1732538791204 (+11 ms)Preparing flush snapshotting stores in 7e66b7e2cad77d6ac6ffe4f9835a87a7 at 1732538791204Finished memstore snapshotting testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732538791204Flushing stores of testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7. at 1732538791204Flushing 7e66b7e2cad77d6ac6ffe4f9835a87a7/a: creating writer at 1732538791204Flushing 7e66b7e2cad77d6ac6ffe4f9835a87a7/a: appending metadata at 1732538791230 (+26 ms)Flushing 7e66b7e2cad77d6ac6ffe4f9835a87a7/a: closing flushed file at 1732538791230Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a4a857a: reopening flushed file at 1732538791249 (+19 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7e66b7e2cad77d6ac6ffe4f9835a87a7 in 57ms, sequenceid=12, compaction requested=false; wal=null at 1732538791261 (+12 ms)Cleaning up temporary data from old regions at 1732538791263 (+2 ms)Region opened successfully at 1732538791270 (+7 ms) 2024-11-25T12:46:31,274 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e66b7e2cad77d6ac6ffe4f9835a87a7, NAME => 'testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:31,274 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732538790590.7e66b7e2cad77d6ac6ffe4f9835a87a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:31,274 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,274 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,276 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,276 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e66b7e2cad77d6ac6ffe4f9835a87a7 columnFamilyName a 2024-11-25T12:46:31,277 DEBUG [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:31,282 DEBUG [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/a/aac1c1ec277e4917a090c4de5e8a5b8e 2024-11-25T12:46:31,282 INFO [StoreOpener-7e66b7e2cad77d6ac6ffe4f9835a87a7-1 {}] regionserver.HStore(327): Store=7e66b7e2cad77d6ac6ffe4f9835a87a7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:31,283 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,283 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,285 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,285 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,285 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,288 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7e66b7e2cad77d6ac6ffe4f9835a87a7 2024-11-25T12:46:31,291 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testDatalossWhenInputError/7e66b7e2cad77d6ac6ffe4f9835a87a7/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-25T12:46:31,292 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7e66b7e2cad77d6ac6ffe4f9835a87a7; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60574301, jitterRate=-0.09737257659435272}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:46:31,292 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7e66b7e2cad77d6ac6ffe4f9835a87a7: Writing region info on filesystem at 1732538791274Initializing all the Stores at 1732538791275 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538791275Cleaning up temporary data from old regions at 1732538791285 (+10 ms)Region opened successfully at 1732538791292 (+7 ms) 2024-11-25T12:46:31,309 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=427 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:55172 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:59544 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:60478 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-225838851-172.17.0.3-1732538750914:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:59470 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:55258 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1256 (was 1174) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=595 (was 595), ProcessCount=11 (was 11), AvailableMemoryMB=1904 (was 1879) - AvailableMemoryMB LEAK? - 2024-11-25T12:46:31,310 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1256 is superior to 1024 2024-11-25T12:46:31,323 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=427, OpenFileDescriptor=1256, MaxFileDescriptor=1048576, SystemLoadAverage=595, ProcessCount=11, AvailableMemoryMB=1899 2024-11-25T12:46:31,323 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1256 is superior to 1024 2024-11-25T12:46:31,338 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:31,340 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:31,340 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:31,343 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78699674, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-78699674, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:31,354 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78699674/hregion-78699674.1732538791343, exclude list is [], retry=0 2024-11-25T12:46:31,357 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:31,357 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:31,357 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:31,359 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78699674/hregion-78699674.1732538791343 2024-11-25T12:46:31,360 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:31,360 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => aefc9e91860146df6a0eadf51e5de324, NAME => 'testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:31,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741921_1099 (size=63) 2024-11-25T12:46:31,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741921_1099 (size=63) 2024-11-25T12:46:31,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741921_1099 (size=63) 2024-11-25T12:46:31,370 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:31,371 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,373 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName a 2024-11-25T12:46:31,373 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:31,374 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:31,374 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,375 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName b 2024-11-25T12:46:31,375 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:31,375 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:31,376 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,377 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName c 2024-11-25T12:46:31,377 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:31,377 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:31,378 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,378 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,379 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,383 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,383 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,384 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:31,386 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,389 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:31,390 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aefc9e91860146df6a0eadf51e5de324; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69047120, jitterRate=0.028882265090942383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:31,391 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aefc9e91860146df6a0eadf51e5de324: Writing region info on filesystem at 1732538791370Initializing all the Stores at 1732538791371 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538791371Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538791371Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538791371Cleaning up temporary data from old regions at 1732538791383 (+12 ms)Region opened successfully at 1732538791391 (+8 ms) 2024-11-25T12:46:31,391 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing aefc9e91860146df6a0eadf51e5de324, disabling compactions & flushes 2024-11-25T12:46:31,391 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:31,391 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:31,391 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. after waiting 0 ms 2024-11-25T12:46:31,391 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:31,392 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:31,392 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for aefc9e91860146df6a0eadf51e5de324: Waiting for close lock at 1732538791391Disabling compacts and flushes for region at 1732538791391Disabling writes for close at 1732538791391Writing region close event to WAL at 1732538791392 (+1 ms)Closed at 1732538791392 2024-11-25T12:46:31,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741920_1098 (size=95) 2024-11-25T12:46:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741920_1098 (size=95) 2024-11-25T12:46:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741920_1098 (size=95) 2024-11-25T12:46:31,399 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:31,399 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78699674:(num 1732538791343) 2024-11-25T12:46:31,400 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:31,402 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:31,417 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402, exclude list is [], retry=0 2024-11-25T12:46:31,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:31,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:31,423 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:31,428 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 2024-11-25T12:46:31,430 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:31,431 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => aefc9e91860146df6a0eadf51e5de324, NAME => 'testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:31,431 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:31,431 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,431 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,433 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,434 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName a 2024-11-25T12:46:31,434 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:31,434 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:31,435 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,435 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName b 2024-11-25T12:46:31,436 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:31,436 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:31,436 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,437 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName c 2024-11-25T12:46:31,437 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:31,438 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:31,438 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,438 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,440 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,441 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,441 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,441 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:31,443 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:31,444 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aefc9e91860146df6a0eadf51e5de324; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63908276, jitterRate=-0.04769247770309448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:31,445 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aefc9e91860146df6a0eadf51e5de324: Writing region info on filesystem at 1732538791431Initializing all the Stores at 1732538791432 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538791432Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538791433 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538791433Cleaning up temporary data from old regions at 1732538791441 (+8 ms)Region opened successfully at 1732538791445 (+4 ms) 2024-11-25T12:46:31,452 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732538791451/Put/seqid=0 2024-11-25T12:46:31,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741923_1101 (size=4875) 2024-11-25T12:46:31,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741923_1101 (size=4875) 2024-11-25T12:46:31,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741923_1101 (size=4875) 2024-11-25T12:46:31,469 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732538791468/Put/seqid=0 2024-11-25T12:46:31,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741924_1102 (size=4875) 2024-11-25T12:46:31,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741924_1102 (size=4875) 2024-11-25T12:46:31,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741924_1102 (size=4875) 2024-11-25T12:46:31,486 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732538791486/Put/seqid=0 2024-11-25T12:46:31,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741925_1103 (size=4875) 2024-11-25T12:46:31,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741925_1103 (size=4875) 2024-11-25T12:46:31,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741925_1103 (size=4875) 2024-11-25T12:46:31,505 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in aefc9e91860146df6a0eadf51e5de324/a 2024-11-25T12:46:31,520 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-25T12:46:31,520 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T12:46:31,520 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in aefc9e91860146df6a0eadf51e5de324/a 2024-11-25T12:46:31,525 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-25T12:46:31,525 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T12:46:31,525 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in aefc9e91860146df6a0eadf51e5de324/a 2024-11-25T12:46:31,533 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-25T12:46:31,533 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T12:46:31,533 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aefc9e91860146df6a0eadf51e5de324 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-25T12:46:31,568 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/.tmp/a/ba854632372d4fa98323450fa8eb9bdf is 55, key is testCompactedBulkLoadedFiles/a:a/1732538791445/Put/seqid=0 2024-11-25T12:46:31,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741926_1104 (size=5107) 2024-11-25T12:46:31,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741926_1104 (size=5107) 2024-11-25T12:46:31,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741926_1104 (size=5107) 2024-11-25T12:46:31,594 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/.tmp/a/ba854632372d4fa98323450fa8eb9bdf 2024-11-25T12:46:31,640 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/.tmp/a/ba854632372d4fa98323450fa8eb9bdf as hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/ba854632372d4fa98323450fa8eb9bdf 2024-11-25T12:46:31,683 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/ba854632372d4fa98323450fa8eb9bdf, entries=1, sequenceid=4, filesize=5.0 K 2024-11-25T12:46:31,688 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for aefc9e91860146df6a0eadf51e5de324 in 154ms, sequenceid=4, compaction requested=false 2024-11-25T12:46:31,688 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for aefc9e91860146df6a0eadf51e5de324: 2024-11-25T12:46:31,694 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_ 2024-11-25T12:46:31,697 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_ 2024-11-25T12:46:31,699 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_ 2024-11-25T12:46:31,704 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile0 into aefc9e91860146df6a0eadf51e5de324/a as hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_ - updating store file list. 2024-11-25T12:46:31,720 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 970d2a48ee1e42bda03d26bc49466a47_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:46:31,720 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_ into aefc9e91860146df6a0eadf51e5de324/a 2024-11-25T12:46:31,721 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile0 into aefc9e91860146df6a0eadf51e5de324/a (new location: hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_) 2024-11-25T12:46:31,725 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile1 into aefc9e91860146df6a0eadf51e5de324/a as hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_ - updating store file list. 2024-11-25T12:46:31,745 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:46:31,745 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_ into aefc9e91860146df6a0eadf51e5de324/a 2024-11-25T12:46:31,745 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile1 into aefc9e91860146df6a0eadf51e5de324/a (new location: hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_) 2024-11-25T12:46:31,747 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile2 into aefc9e91860146df6a0eadf51e5de324/a as hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_ - updating store file list. 2024-11-25T12:46:31,758 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8780466b6e2247c68ed885945d71b0f6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:46:31,759 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_ into aefc9e91860146df6a0eadf51e5de324/a 2024-11-25T12:46:31,759 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35793/hbase/testCompactedBulkLoadedFiles/hfile2 into aefc9e91860146df6a0eadf51e5de324/a (new location: hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_) 2024-11-25T12:46:31,774 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:46:31,775 DEBUG [Time-limited test {}] regionserver.HStore(1541): aefc9e91860146df6a0eadf51e5de324/a is initiating major compaction (all files) 2024-11-25T12:46:31,775 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of aefc9e91860146df6a0eadf51e5de324/a in testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:31,775 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/ba854632372d4fa98323450fa8eb9bdf, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_] into tmpdir=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/.tmp, totalSize=19.3 K 2024-11-25T12:46:31,777 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ba854632372d4fa98323450fa8eb9bdf, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732538791445 2024-11-25T12:46:31,777 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 970d2a48ee1e42bda03d26bc49466a47_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T12:46:31,778 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T12:46:31,778 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8780466b6e2247c68ed885945d71b0f6_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T12:46:31,802 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/.tmp/a/3108e1fc78994998ab348913f8901e38 is 55, key is testCompactedBulkLoadedFiles/a:a/1732538791445/Put/seqid=0 2024-11-25T12:46:31,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741927_1105 (size=6154) 2024-11-25T12:46:31,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741927_1105 (size=6154) 2024-11-25T12:46:31,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741927_1105 (size=6154) 2024-11-25T12:46:31,831 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/.tmp/a/3108e1fc78994998ab348913f8901e38 as hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/3108e1fc78994998ab348913f8901e38 2024-11-25T12:46:31,840 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in aefc9e91860146df6a0eadf51e5de324/a of aefc9e91860146df6a0eadf51e5de324 into 3108e1fc78994998ab348913f8901e38(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:46:31,840 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for aefc9e91860146df6a0eadf51e5de324: 2024-11-25T12:46:31,840 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-25T12:46:31,840 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-25T12:46:31,893 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402, size=0 (0bytes) 2024-11-25T12:46:31,893 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 might be still open, length is 0 2024-11-25T12:46:31,893 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 2024-11-25T12:46:31,894 WARN [IPC Server handler 3 on default port 35793 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-25T12:46:31,894 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 after 1ms 2024-11-25T12:46:33,276 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:59584 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59584 dst: /127.0.0.1:34959 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34959 remote=/127.0.0.1:59584]. Total timeout mills is 60000, 58564 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:33,276 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:60518 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:33377:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60518 dst: /127.0.0.1:33377 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:33,278 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:55290 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:35653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55290 dst: /127.0.0.1:35653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:33,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741922_1106 (size=1171) 2024-11-25T12:46:33,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741922_1106 (size=1171) 2024-11-25T12:46:35,744 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:46:35,895 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 after 4002ms 2024-11-25T12:46:35,898 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:35,898 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 took 4005ms 2024-11-25T12:46:35,903 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402; continuing. 2024-11-25T12:46:35,903 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 so closing down 2024-11-25T12:46:35,903 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:35,905 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538791402.temp 2024-11-25T12:46:35,907 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000003-wal.1732538791402.temp 2024-11-25T12:46:35,908 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:35,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741928_1107 (size=548) 2024-11-25T12:46:35,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741928_1107 (size=548) 2024-11-25T12:46:35,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741928_1107 (size=548) 2024-11-25T12:46:35,919 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000003-wal.1732538791402.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:35,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000003-wal.1732538791402.temp to hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000008 2024-11-25T12:46:35,921 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 20 ms; skipped=3; WAL=hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T12:46:35,921 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402, journal: Splitting hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402, size=0 (0bytes) at 1732538791893Finishing writing output for hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 so closing down at 1732538795903 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000003-wal.1732538791402.temp at 1732538795907 (+4 ms)3 split writer threads finished at 1732538795908 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000003-wal.1732538791402.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732538795919 (+11 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000003-wal.1732538791402.temp to hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000008 at 1732538795921 (+2 ms)Processed 5 edits across 1 Regions in 20 ms; skipped=3; WAL=hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402, size=0, length=0, corrupted=false, cancelled=false at 1732538795921 2024-11-25T12:46:35,923 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538791402 2024-11-25T12:46:35,924 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000008 2024-11-25T12:46:35,924 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:35,926 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:35,939 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538795926, exclude list is [], retry=0 2024-11-25T12:46:35,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:35,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:35,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:35,947 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538795926 2024-11-25T12:46:35,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:35,948 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => aefc9e91860146df6a0eadf51e5de324, NAME => 'testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:35,948 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:35,948 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:35,948 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:35,950 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:35,951 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName a 2024-11-25T12:46:35,952 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:35,970 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/3108e1fc78994998ab348913f8901e38 2024-11-25T12:46:35,977 DEBUG [StoreFileOpener-aefc9e91860146df6a0eadf51e5de324-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:46:35,977 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_ 2024-11-25T12:46:35,984 DEBUG [StoreFileOpener-aefc9e91860146df6a0eadf51e5de324-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8780466b6e2247c68ed885945d71b0f6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:46:35,984 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_ 2024-11-25T12:46:35,989 DEBUG [StoreFileOpener-aefc9e91860146df6a0eadf51e5de324-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 970d2a48ee1e42bda03d26bc49466a47_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:46:35,989 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_ 2024-11-25T12:46:35,993 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/ba854632372d4fa98323450fa8eb9bdf 2024-11-25T12:46:35,993 WARN [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@76ebd70 2024-11-25T12:46:35,993 WARN [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@76ebd70 2024-11-25T12:46:35,993 WARN [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@76ebd70 2024-11-25T12:46:35,994 WARN [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/ba854632372d4fa98323450fa8eb9bdf from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@76ebd70 2024-11-25T12:46:35,994 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/ba854632372d4fa98323450fa8eb9bdf] to archive 2024-11-25T12:46:35,995 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:46:35,996 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_ to hdfs://localhost:35793/hbase/archive/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_ 2024-11-25T12:46:35,997 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_ to hdfs://localhost:35793/hbase/archive/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/8780466b6e2247c68ed885945d71b0f6_SeqId_4_ 2024-11-25T12:46:35,999 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_ to hdfs://localhost:35793/hbase/archive/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/970d2a48ee1e42bda03d26bc49466a47_SeqId_4_ 2024-11-25T12:46:36,000 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/ba854632372d4fa98323450fa8eb9bdf to hdfs://localhost:35793/hbase/archive/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/a/ba854632372d4fa98323450fa8eb9bdf 2024-11-25T12:46:36,000 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,000 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:36,001 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName b 2024-11-25T12:46:36,001 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,001 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,001 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:36,002 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefc9e91860146df6a0eadf51e5de324 columnFamilyName c 2024-11-25T12:46:36,002 DEBUG [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,003 INFO [StoreOpener-aefc9e91860146df6a0eadf51e5de324-1 {}] regionserver.HStore(327): Store=aefc9e91860146df6a0eadf51e5de324/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,003 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:36,003 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:36,005 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:36,005 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000008 2024-11-25T12:46:36,007 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:36,008 DEBUG [Time-limited test {}] regionserver.HRegion(5836): aefc9e91860146df6a0eadf51e5de324 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "aefc9e91860146df6a0eadf51e5de324" family_name: "a" compaction_input: "ba854632372d4fa98323450fa8eb9bdf" compaction_input: "970d2a48ee1e42bda03d26bc49466a47_SeqId_4_" compaction_input: "6f1b60d021c14534bc5cf83149ca3e89_SeqId_4_" compaction_input: "8780466b6e2247c68ed885945d71b0f6_SeqId_4_" compaction_output: "3108e1fc78994998ab348913f8901e38" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-25T12:46:36,008 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-25T12:46:36,009 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000008 2024-11-25T12:46:36,009 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/0000000000000000008 2024-11-25T12:46:36,011 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:36,011 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:36,011 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:36,012 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aefc9e91860146df6a0eadf51e5de324 2024-11-25T12:46:36,014 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testCompactedBulkLoadedFiles/aefc9e91860146df6a0eadf51e5de324/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-25T12:46:36,015 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aefc9e91860146df6a0eadf51e5de324; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66169083, jitterRate=-0.014003828167915344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:36,015 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aefc9e91860146df6a0eadf51e5de324: Writing region info on filesystem at 1732538795948Initializing all the Stores at 1732538795950 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538795950Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538795950Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538795950Cleaning up temporary data from old regions at 1732538796011 (+61 ms)Region opened successfully at 1732538796015 (+4 ms) 2024-11-25T12:46:36,018 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing aefc9e91860146df6a0eadf51e5de324, disabling compactions & flushes 2024-11-25T12:46:36,018 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:36,018 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:36,018 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. after waiting 0 ms 2024-11-25T12:46:36,018 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:36,019 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732538791338.aefc9e91860146df6a0eadf51e5de324. 2024-11-25T12:46:36,019 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for aefc9e91860146df6a0eadf51e5de324: Waiting for close lock at 1732538796017Disabling compacts and flushes for region at 1732538796017Disabling writes for close at 1732538796018 (+1 ms)Writing region close event to WAL at 1732538796019 (+1 ms)Closed at 1732538796019 2024-11-25T12:46:36,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741929_1108 (size=95) 2024-11-25T12:46:36,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741929_1108 (size=95) 2024-11-25T12:46:36,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741929_1108 (size=95) 2024-11-25T12:46:36,024 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:36,024 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538795926) 2024-11-25T12:46:36,040 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=437 (was 427) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1231096508_22 at /127.0.0.1:54006 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:35793 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35793 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1231096508_22 at /127.0.0.1:42228 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1231096508_22 at /127.0.0.1:33108 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1336 (was 1256) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=555 (was 595), ProcessCount=11 (was 11), AvailableMemoryMB=1869 (was 1899) 2024-11-25T12:46:36,040 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1336 is superior to 1024 2024-11-25T12:46:36,055 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1336, MaxFileDescriptor=1048576, SystemLoadAverage=555, ProcessCount=11, AvailableMemoryMB=1865 2024-11-25T12:46:36,055 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1336 is superior to 1024 2024-11-25T12:46:36,076 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:36,078 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:46:36,079 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:46:36,082 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11744368, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/hregion-11744368, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:36,101 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11744368/hregion-11744368.1732538796082, exclude list is [], retry=0 2024-11-25T12:46:36,104 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:36,104 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:36,104 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:36,107 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11744368/hregion-11744368.1732538796082 2024-11-25T12:46:36,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:36,109 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 78cf828cce300636cd5ff84b71ad4d6f, NAME => 'testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35793/hbase 2024-11-25T12:46:36,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-25T12:46:36,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:36,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-25T12:46:36,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-25T12:46:36,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741931_1110 (size=67) 2024-11-25T12:46:36,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741931_1110 (size=67) 2024-11-25T12:46:36,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741931_1110 (size=67) 2024-11-25T12:46:36,130 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:36,131 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,134 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName a 2024-11-25T12:46:36,135 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,136 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,136 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,140 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName b 2024-11-25T12:46:36,140 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,141 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,141 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,142 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName c 2024-11-25T12:46:36,142 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,143 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,143 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,144 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,145 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,146 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,146 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,147 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:36,148 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,156 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:46:36,157 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 78cf828cce300636cd5ff84b71ad4d6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62363864, jitterRate=-0.07070600986480713}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:36,157 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 78cf828cce300636cd5ff84b71ad4d6f: Writing region info on filesystem at 1732538796130Initializing all the Stores at 1732538796131 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796131Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796131Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796131Cleaning up temporary data from old regions at 1732538796146 (+15 ms)Region opened successfully at 1732538796157 (+11 ms) 2024-11-25T12:46:36,157 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 78cf828cce300636cd5ff84b71ad4d6f, disabling compactions & flushes 2024-11-25T12:46:36,157 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,157 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,157 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. after waiting 0 ms 2024-11-25T12:46:36,157 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,159 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,160 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 78cf828cce300636cd5ff84b71ad4d6f: Waiting for close lock at 1732538796157Disabling compacts and flushes for region at 1732538796157Disabling writes for close at 1732538796157Writing region close event to WAL at 1732538796159 (+2 ms)Closed at 1732538796159 2024-11-25T12:46:36,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741930_1109 (size=95) 2024-11-25T12:46:36,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741930_1109 (size=95) 2024-11-25T12:46:36,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741930_1109 (size=95) 2024-11-25T12:46:36,167 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:36,167 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11744368:(num 1732538796082) 2024-11-25T12:46:36,168 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:36,169 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:36,189 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170, exclude list is [], retry=0 2024-11-25T12:46:36,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:36,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:36,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:36,195 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170 2024-11-25T12:46:36,196 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:40413:40413)] 2024-11-25T12:46:36,196 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 78cf828cce300636cd5ff84b71ad4d6f, NAME => 'testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:36,196 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:36,196 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,196 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,198 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,199 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName a 2024-11-25T12:46:36,199 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,200 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,200 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,201 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName b 2024-11-25T12:46:36,201 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,201 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,202 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,202 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName c 2024-11-25T12:46:36,202 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,203 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,203 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,208 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,212 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:36,213 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,214 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 78cf828cce300636cd5ff84b71ad4d6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65733109, jitterRate=-0.020500347018241882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:36,214 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 78cf828cce300636cd5ff84b71ad4d6f: Writing region info on filesystem at 1732538796196Initializing all the Stores at 1732538796197 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796197Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796198 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796198Cleaning up temporary data from old regions at 1732538796211 (+13 ms)Region opened successfully at 1732538796214 (+3 ms) 2024-11-25T12:46:36,242 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 78cf828cce300636cd5ff84b71ad4d6f 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-25T12:46:36,261 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/a/e39f1f02cf6e4a41b4234695b2e1a8af is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732538796214/Put/seqid=0 2024-11-25T12:46:36,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741933_1112 (size=5958) 2024-11-25T12:46:36,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741933_1112 (size=5958) 2024-11-25T12:46:36,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741933_1112 (size=5958) 2024-11-25T12:46:36,287 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/a/e39f1f02cf6e4a41b4234695b2e1a8af 2024-11-25T12:46:36,300 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/a/e39f1f02cf6e4a41b4234695b2e1a8af as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/a/e39f1f02cf6e4a41b4234695b2e1a8af 2024-11-25T12:46:36,315 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/a/e39f1f02cf6e4a41b4234695b2e1a8af, entries=10, sequenceid=13, filesize=5.8 K 2024-11-25T12:46:36,318 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 78cf828cce300636cd5ff84b71ad4d6f in 76ms, sequenceid=13, compaction requested=false 2024-11-25T12:46:36,318 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 78cf828cce300636cd5ff84b71ad4d6f: 2024-11-25T12:46:36,348 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 78cf828cce300636cd5ff84b71ad4d6f, disabling compactions & flushes 2024-11-25T12:46:36,348 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,348 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,348 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. after waiting 0 ms 2024-11-25T12:46:36,348 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,351 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,351 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:36,351 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 78cf828cce300636cd5ff84b71ad4d6f: Waiting for close lock at 1732538796348Disabling compacts and flushes for region at 1732538796348Disabling writes for close at 1732538796348Writing region close event to WAL at 1732538796351 (+3 ms)Closed at 1732538796351 2024-11-25T12:46:36,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741932_1111 (size=3346) 2024-11-25T12:46:36,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741932_1111 (size=3346) 2024-11-25T12:46:36,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741932_1111 (size=3346) 2024-11-25T12:46:36,382 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170, size=3.3 K (3346bytes) 2024-11-25T12:46:36,382 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170 2024-11-25T12:46:36,383 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170 after 1ms 2024-11-25T12:46:36,410 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:36,411 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170 took 29ms 2024-11-25T12:46:36,413 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170 so closing down 2024-11-25T12:46:36,413 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:36,414 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538796170.temp 2024-11-25T12:46:36,415 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000003-wal.1732538796170.temp 2024-11-25T12:46:36,415 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:36,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741934_1113 (size=2944) 2024-11-25T12:46:36,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741934_1113 (size=2944) 2024-11-25T12:46:36,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741934_1113 (size=2944) 2024-11-25T12:46:36,421 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000003-wal.1732538796170.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:36,422 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000003-wal.1732538796170.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000035 2024-11-25T12:46:36,423 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170, size=3.3 K, length=3346, corrupted=false, cancelled=false 2024-11-25T12:46:36,423 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170, journal: Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170, size=3.3 K (3346bytes) at 1732538796382Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170 so closing down at 1732538796413 (+31 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000003-wal.1732538796170.temp at 1732538796415 (+2 ms)3 split writer threads finished at 1732538796416 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000003-wal.1732538796170.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732538796421 (+5 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000003-wal.1732538796170.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000035 at 1732538796422 (+1 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170, size=3.3 K, length=3346, corrupted=false, cancelled=false at 1732538796423 (+1 ms) 2024-11-25T12:46:36,424 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796170 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538796170 2024-11-25T12:46:36,425 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000035 2024-11-25T12:46:36,425 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:36,426 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:36,442 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427, exclude list is [], retry=0 2024-11-25T12:46:36,444 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:36,444 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:36,445 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:36,446 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 2024-11-25T12:46:36,446 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43987:43987),(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547)] 2024-11-25T12:46:36,447 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 78cf828cce300636cd5ff84b71ad4d6f, NAME => 'testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:46:36,447 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:36,447 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,447 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,448 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,449 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName a 2024-11-25T12:46:36,449 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,455 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/a/e39f1f02cf6e4a41b4234695b2e1a8af 2024-11-25T12:46:36,455 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,455 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,456 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName b 2024-11-25T12:46:36,456 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,457 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,457 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,458 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName c 2024-11-25T12:46:36,458 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:36,458 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:36,458 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,459 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,460 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,461 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000035 2024-11-25T12:46:36,464 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:36,465 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000035 2024-11-25T12:46:36,466 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 78cf828cce300636cd5ff84b71ad4d6f 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-25T12:46:36,481 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/b/c4c0e38936d1465c8e8d3c36fd0a264c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732538796318/Put/seqid=0 2024-11-25T12:46:36,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741936_1115 (size=5958) 2024-11-25T12:46:36,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741936_1115 (size=5958) 2024-11-25T12:46:36,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741936_1115 (size=5958) 2024-11-25T12:46:36,488 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/b/c4c0e38936d1465c8e8d3c36fd0a264c 2024-11-25T12:46:36,507 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/c/69181cf840df4b5586b6ce78bd392138 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732538796327/Put/seqid=0 2024-11-25T12:46:36,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741937_1116 (size=5958) 2024-11-25T12:46:36,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741937_1116 (size=5958) 2024-11-25T12:46:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741937_1116 (size=5958) 2024-11-25T12:46:36,914 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/c/69181cf840df4b5586b6ce78bd392138 2024-11-25T12:46:36,921 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/b/c4c0e38936d1465c8e8d3c36fd0a264c as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/b/c4c0e38936d1465c8e8d3c36fd0a264c 2024-11-25T12:46:36,927 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/b/c4c0e38936d1465c8e8d3c36fd0a264c, entries=10, sequenceid=35, filesize=5.8 K 2024-11-25T12:46:36,929 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/c/69181cf840df4b5586b6ce78bd392138 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/c/69181cf840df4b5586b6ce78bd392138 2024-11-25T12:46:36,935 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/c/69181cf840df4b5586b6ce78bd392138, entries=10, sequenceid=35, filesize=5.8 K 2024-11-25T12:46:36,935 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 78cf828cce300636cd5ff84b71ad4d6f in 470ms, sequenceid=35, compaction requested=false; wal=null 2024-11-25T12:46:36,936 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000035 2024-11-25T12:46:36,937 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,937 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,937 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:36,939 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:36,940 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-25T12:46:36,941 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 78cf828cce300636cd5ff84b71ad4d6f; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60631872, jitterRate=-0.09651470184326172}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:36,941 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 78cf828cce300636cd5ff84b71ad4d6f: Writing region info on filesystem at 1732538796447Initializing all the Stores at 1732538796448 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796448Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796448Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538796448Obtaining lock to block concurrent updates at 1732538796466 (+18 ms)Preparing flush snapshotting stores in 78cf828cce300636cd5ff84b71ad4d6f at 1732538796466Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732538796466Flushing stores of testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. at 1732538796466Flushing 78cf828cce300636cd5ff84b71ad4d6f/b: creating writer at 1732538796466Flushing 78cf828cce300636cd5ff84b71ad4d6f/b: appending metadata at 1732538796480 (+14 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/b: closing flushed file at 1732538796481 (+1 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/c: creating writer at 1732538796492 (+11 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/c: appending metadata at 1732538796506 (+14 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/c: closing flushed file at 1732538796506Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12b31463: reopening flushed file at 1732538796920 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62781a22: reopening flushed file at 1732538796928 (+8 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 78cf828cce300636cd5ff84b71ad4d6f in 470ms, sequenceid=35, compaction requested=false; wal=null at 1732538796935 (+7 ms)Cleaning up temporary data from old regions at 1732538796937 (+2 ms)Region opened successfully at 1732538796941 (+4 ms) 2024-11-25T12:46:37,016 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427, size=0 (0bytes) 2024-11-25T12:46:37,016 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 might be still open, length is 0 2024-11-25T12:46:37,016 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 2024-11-25T12:46:37,017 WARN [IPC Server handler 3 on default port 35793 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-25T12:46:37,017 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 after 1ms 2024-11-25T12:46:39,281 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:42314 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:35653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42314 dst: /127.0.0.1:35653 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35653 remote=/127.0.0.1:42314]. Total timeout mills is 60000, 57691 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:39,282 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:33150 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33150 dst: /127.0.0.1:34959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:39,282 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-896067521_22 at /127.0.0.1:54052 [Receiving block BP-225838851-172.17.0.3-1732538750914:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:33377:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54052 dst: /127.0.0.1:33377 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:39,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741935_1117 (size=2936) 2024-11-25T12:46:39,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741935_1117 (size=2936) 2024-11-25T12:46:39,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741935_1117 (size=2936) 2024-11-25T12:46:41,019 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 after 4003ms 2024-11-25T12:46:41,025 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:41,025 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 took 4009ms 2024-11-25T12:46:41,028 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427; continuing. 2024-11-25T12:46:41,028 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 so closing down 2024-11-25T12:46:41,028 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:46:41,029 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732538796427.temp 2024-11-25T12:46:41,031 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000037-wal.1732538796427.temp 2024-11-25T12:46:41,031 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:46:41,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741938_1118 (size=2944) 2024-11-25T12:46:41,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741938_1118 (size=2944) 2024-11-25T12:46:41,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741938_1118 (size=2944) 2024-11-25T12:46:41,038 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000037-wal.1732538796427.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T12:46:41,039 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000037-wal.1732538796427.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000066 2024-11-25T12:46:41,039 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T12:46:41,040 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427, journal: Splitting hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427, size=0 (0bytes) at 1732538797016Finishing writing output for hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 so closing down at 1732538801028 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000037-wal.1732538796427.temp at 1732538801031 (+3 ms)3 split writer threads finished at 1732538801031Closed recovered edits writer path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000037-wal.1732538796427.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732538801038 (+7 ms)Rename recovered edits hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000037-wal.1732538796427.temp to hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000066 at 1732538801039 (+1 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427, size=0, length=0, corrupted=false, cancelled=false at 1732538801039 2024-11-25T12:46:41,041 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 to hdfs://localhost:35793/hbase/oldWALs/wal.1732538796427 2024-11-25T12:46:41,042 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000066 2024-11-25T12:46:41,042 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:46:41,043 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35793/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075, archiveDir=hdfs://localhost:35793/hbase/oldWALs, maxLogs=32 2024-11-25T12:46:41,056 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538801044, exclude list is [], retry=0 2024-11-25T12:46:41,058 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33377,DS-360fdf6d-0499-4d4c-bea9-178df09bb758,DISK] 2024-11-25T12:46:41,058 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34959,DS-0f0e0504-2a59-4b74-bb48-5c34ec0c5cd5,DISK] 2024-11-25T12:46:41,059 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35653,DS-6f16a029-d49d-4918-a940-498eea8876d4,DISK] 2024-11-25T12:46:41,063 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538801044 2024-11-25T12:46:41,064 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40413:40413),(127.0.0.1/127.0.0.1:33547:33547),(127.0.0.1/127.0.0.1:43987:43987)] 2024-11-25T12:46:41,064 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:46:41,065 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,066 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName a 2024-11-25T12:46:41,066 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:41,072 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/a/e39f1f02cf6e4a41b4234695b2e1a8af 2024-11-25T12:46:41,072 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:41,072 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,073 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName b 2024-11-25T12:46:41,073 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:41,078 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/b/c4c0e38936d1465c8e8d3c36fd0a264c 2024-11-25T12:46:41,078 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:41,078 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,079 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cf828cce300636cd5ff84b71ad4d6f columnFamilyName c 2024-11-25T12:46:41,079 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:46:41,083 DEBUG [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/c/69181cf840df4b5586b6ce78bd392138 2024-11-25T12:46:41,083 INFO [StoreOpener-78cf828cce300636cd5ff84b71ad4d6f-1 {}] regionserver.HStore(327): Store=78cf828cce300636cd5ff84b71ad4d6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:46:41,083 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,084 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,086 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,086 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000066 2024-11-25T12:46:41,088 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-25T12:46:41,092 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000066 2024-11-25T12:46:41,092 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 78cf828cce300636cd5ff84b71ad4d6f 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-25T12:46:41,106 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/a/76a5ad1da7e1448cbc4d4263c4173ccf is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732538796947/Put/seqid=0 2024-11-25T12:46:41,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741940_1120 (size=5958) 2024-11-25T12:46:41,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741940_1120 (size=5958) 2024-11-25T12:46:41,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741940_1120 (size=5958) 2024-11-25T12:46:41,112 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/a/76a5ad1da7e1448cbc4d4263c4173ccf 2024-11-25T12:46:41,129 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/b/1b31fac20bed4dafb4915a543b04f4a7 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732538796954/Put/seqid=0 2024-11-25T12:46:41,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741941_1121 (size=5958) 2024-11-25T12:46:41,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741941_1121 (size=5958) 2024-11-25T12:46:41,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741941_1121 (size=5958) 2024-11-25T12:46:41,135 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/b/1b31fac20bed4dafb4915a543b04f4a7 2024-11-25T12:46:41,153 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/c/e502292b90154da9a2c1f8939682492d is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732538796961/Put/seqid=0 2024-11-25T12:46:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741942_1122 (size=5958) 2024-11-25T12:46:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741942_1122 (size=5958) 2024-11-25T12:46:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741942_1122 (size=5958) 2024-11-25T12:46:41,159 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/c/e502292b90154da9a2c1f8939682492d 2024-11-25T12:46:41,164 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/a/76a5ad1da7e1448cbc4d4263c4173ccf as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/a/76a5ad1da7e1448cbc4d4263c4173ccf 2024-11-25T12:46:41,169 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/a/76a5ad1da7e1448cbc4d4263c4173ccf, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T12:46:41,171 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/b/1b31fac20bed4dafb4915a543b04f4a7 as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/b/1b31fac20bed4dafb4915a543b04f4a7 2024-11-25T12:46:41,176 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/b/1b31fac20bed4dafb4915a543b04f4a7, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T12:46:41,177 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/.tmp/c/e502292b90154da9a2c1f8939682492d as hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/c/e502292b90154da9a2c1f8939682492d 2024-11-25T12:46:41,182 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/c/e502292b90154da9a2c1f8939682492d, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T12:46:41,182 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 78cf828cce300636cd5ff84b71ad4d6f in 90ms, sequenceid=66, compaction requested=false; wal=null 2024-11-25T12:46:41,183 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/0000000000000000066 2024-11-25T12:46:41,184 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,184 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,185 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:46:41,186 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 78cf828cce300636cd5ff84b71ad4d6f 2024-11-25T12:46:41,188 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/hbase/data/default/testReplayEditsWrittenViaHRegion/78cf828cce300636cd5ff84b71ad4d6f/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-25T12:46:41,189 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 78cf828cce300636cd5ff84b71ad4d6f; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66173743, jitterRate=-0.013934388756752014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:46:41,189 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 78cf828cce300636cd5ff84b71ad4d6f: Writing region info on filesystem at 1732538801064Initializing all the Stores at 1732538801065 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538801065Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538801065Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538801065Obtaining lock to block concurrent updates at 1732538801092 (+27 ms)Preparing flush snapshotting stores in 78cf828cce300636cd5ff84b71ad4d6f at 1732538801092Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732538801092Flushing stores of testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. at 1732538801092Flushing 78cf828cce300636cd5ff84b71ad4d6f/a: creating writer at 1732538801092Flushing 78cf828cce300636cd5ff84b71ad4d6f/a: appending metadata at 1732538801106 (+14 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/a: closing flushed file at 1732538801106Flushing 78cf828cce300636cd5ff84b71ad4d6f/b: creating writer at 1732538801116 (+10 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/b: appending metadata at 1732538801128 (+12 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/b: closing flushed file at 1732538801128Flushing 78cf828cce300636cd5ff84b71ad4d6f/c: creating writer at 1732538801139 (+11 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/c: appending metadata at 1732538801152 (+13 ms)Flushing 78cf828cce300636cd5ff84b71ad4d6f/c: closing flushed file at 1732538801152Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b4c68dc: reopening flushed file at 1732538801163 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a67e911: reopening flushed file at 1732538801169 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@86e6730: reopening flushed file at 1732538801176 (+7 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 78cf828cce300636cd5ff84b71ad4d6f in 90ms, sequenceid=66, compaction requested=false; wal=null at 1732538801182 (+6 ms)Cleaning up temporary data from old regions at 1732538801184 (+2 ms)Region opened successfully at 1732538801189 (+5 ms) 2024-11-25T12:46:41,200 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 78cf828cce300636cd5ff84b71ad4d6f, disabling compactions & flushes 2024-11-25T12:46:41,200 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:41,200 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:41,200 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. after waiting 0 ms 2024-11-25T12:46:41,200 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:41,202 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538796077.78cf828cce300636cd5ff84b71ad4d6f. 2024-11-25T12:46:41,202 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 78cf828cce300636cd5ff84b71ad4d6f: Waiting for close lock at 1732538801200Disabling compacts and flushes for region at 1732538801200Disabling writes for close at 1732538801200Writing region close event to WAL at 1732538801202 (+2 ms)Closed at 1732538801202 2024-11-25T12:46:41,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741939_1119 (size=95) 2024-11-25T12:46:41,206 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538801044 not finished, retry = 0 2024-11-25T12:46:41,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741939_1119 (size=95) 2024-11-25T12:46:41,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741939_1119 (size=95) 2024-11-25T12:46:41,207 INFO [master/1ff8009e463e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T12:46:41,207 INFO [master/1ff8009e463e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T12:46:41,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:46:41,309 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538801044) 2024-11-25T12:46:41,325 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=441 (was 437) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1477091796_22 at /127.0.0.1:33182 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:35793 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:35793 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1477091796_22 at /127.0.0.1:42348 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1477091796_22 at /127.0.0.1:54072 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1400 (was 1336) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=535 (was 555), ProcessCount=11 (was 11), AvailableMemoryMB=1790 (was 1865) 2024-11-25T12:46:41,325 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1400 is superior to 1024 2024-11-25T12:46:41,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T12:46:41,325 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:46:41,325 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:46:41,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:41,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:41,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:46:41,326 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T12:46:41,326 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=634595892, stopped=false 2024-11-25T12:46:41,327 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1ff8009e463e,36927,1732538755691 2024-11-25T12:46:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T12:46:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T12:46:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T12:46:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:46:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:46:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:46:41,370 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T12:46:41,371 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:46:41,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:46:41,371 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:46:41,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:46:41,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:41,372 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:46:41,372 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ff8009e463e,43295,1732538756657' ***** 2024-11-25T12:46:41,372 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T12:46:41,372 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ff8009e463e,35205,1732538757028' ***** 2024-11-25T12:46:41,372 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T12:46:41,373 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T12:46:41,373 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T12:46:41,373 INFO [RS:0;1ff8009e463e:43295 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T12:46:41,373 INFO [RS:0;1ff8009e463e:43295 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T12:46:41,373 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T12:46:41,373 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(3091): Received CLOSE for c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:41,373 INFO [RS:2;1ff8009e463e:35205 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T12:46:41,373 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T12:46:41,373 INFO [RS:2;1ff8009e463e:35205 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T12:46:41,373 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(959): stopping server 1ff8009e463e,43295,1732538756657 2024-11-25T12:46:41,373 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(959): stopping server 1ff8009e463e,35205,1732538757028 2024-11-25T12:46:41,373 INFO [RS:2;1ff8009e463e:35205 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:46:41,373 INFO [RS:0;1ff8009e463e:43295 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:46:41,373 INFO [RS:0;1ff8009e463e:43295 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1ff8009e463e:43295. 2024-11-25T12:46:41,373 INFO [RS:2;1ff8009e463e:35205 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1ff8009e463e:35205. 2024-11-25T12:46:41,373 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c6d3a9a4a6e575e3ff59df3723af337c, disabling compactions & flushes 2024-11-25T12:46:41,373 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:41,373 DEBUG [RS:2;1ff8009e463e:35205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:46:41,373 DEBUG [RS:0;1ff8009e463e:43295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:46:41,373 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:41,373 DEBUG [RS:2;1ff8009e463e:35205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:41,373 DEBUG [RS:0;1ff8009e463e:43295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:41,373 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. after waiting 0 ms 2024-11-25T12:46:41,373 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:41,373 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T12:46:41,373 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T12:46:41,374 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T12:46:41,374 DEBUG [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1325): Online Regions={c6d3a9a4a6e575e3ff59df3723af337c=testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c.} 2024-11-25T12:46:41,374 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T12:46:41,374 DEBUG [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1351): Waiting on c6d3a9a4a6e575e3ff59df3723af337c 2024-11-25T12:46:41,374 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T12:46:41,374 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T12:46:41,374 DEBUG [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T12:46:41,374 DEBUG [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T12:46:41,374 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T12:46:41,374 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T12:46:41,374 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T12:46:41,374 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T12:46:41,374 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T12:46:41,374 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-25T12:46:41,379 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c6d3a9a4a6e575e3ff59df3723af337c/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-25T12:46:41,379 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:41,379 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c6d3a9a4a6e575e3ff59df3723af337c: Waiting for close lock at 1732538801373Running coprocessor pre-close hooks at 1732538801373Disabling compacts and flushes for region at 1732538801373Disabling writes for close at 1732538801373Writing region close event to WAL at 1732538801375 (+2 ms)Running coprocessor post-close hooks at 1732538801379 (+4 ms)Closed at 1732538801379 2024-11-25T12:46:41,379 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c. 2024-11-25T12:46:41,380 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/info/02a9713cec8741c9a03d84ad7ba556bd is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732538774680.c6d3a9a4a6e575e3ff59df3723af337c./info:regioninfo/1732538779181/Put/seqid=0 2024-11-25T12:46:41,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741943_1123 (size=6778) 2024-11-25T12:46:41,384 INFO [regionserver/1ff8009e463e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:46:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741943_1123 (size=6778) 2024-11-25T12:46:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741943_1123 (size=6778) 2024-11-25T12:46:41,385 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/info/02a9713cec8741c9a03d84ad7ba556bd 2024-11-25T12:46:41,390 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/.tmp/info/02a9713cec8741c9a03d84ad7ba556bd as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/info/02a9713cec8741c9a03d84ad7ba556bd 2024-11-25T12:46:41,395 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/info/02a9713cec8741c9a03d84ad7ba556bd, entries=8, sequenceid=23, filesize=6.6 K 2024-11-25T12:46:41,396 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 22ms, sequenceid=23, compaction requested=false 2024-11-25T12:46:41,402 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-25T12:46:41,403 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T12:46:41,403 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T12:46:41,403 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732538801374Running coprocessor pre-close hooks at 1732538801374Disabling compacts and flushes for region at 1732538801374Disabling writes for close at 1732538801374Obtaining lock to block concurrent updates at 1732538801374Preparing flush snapshotting stores in 1588230740 at 1732538801374Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732538801375 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732538801375Flushing 1588230740/info: creating writer at 1732538801375Flushing 1588230740/info: appending metadata at 1732538801379 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732538801379Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d4cca22: reopening flushed file at 1732538801390 (+11 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 22ms, sequenceid=23, compaction requested=false at 1732538801396 (+6 ms)Writing region close event to WAL at 1732538801399 (+3 ms)Running coprocessor post-close hooks at 1732538801403 (+4 ms)Closed at 1732538801403 2024-11-25T12:46:41,403 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T12:46:41,414 INFO [regionserver/1ff8009e463e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:46:41,574 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(976): stopping server 1ff8009e463e,43295,1732538756657; all regions closed. 2024-11-25T12:46:41,574 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(976): stopping server 1ff8009e463e,35205,1732538757028; all regions closed. 2024-11-25T12:46:41,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741833_1009 (size=2185) 2024-11-25T12:46:41,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741833_1009 (size=2185) 2024-11-25T12:46:41,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741893_1071 (size=1674) 2024-11-25T12:46:41,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741893_1071 (size=1674) 2024-11-25T12:46:41,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741893_1071 (size=1674) 2024-11-25T12:46:41,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741833_1009 (size=2185) 2024-11-25T12:46:41,587 DEBUG [RS:0;1ff8009e463e:43295 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs 2024-11-25T12:46:41,587 DEBUG [RS:2;1ff8009e463e:35205 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs 2024-11-25T12:46:41,587 INFO [RS:0;1ff8009e463e:43295 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ff8009e463e%2C43295%2C1732538756657:(num 1732538759942) 2024-11-25T12:46:41,587 INFO [RS:2;1ff8009e463e:35205 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ff8009e463e%2C35205%2C1732538757028.meta:.meta(num 1732538778249) 2024-11-25T12:46:41,587 DEBUG [RS:0;1ff8009e463e:43295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:41,587 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:46:41,587 INFO [RS:0;1ff8009e463e:43295 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:46:41,587 INFO [RS:0;1ff8009e463e:43295 {}] hbase.ChoreService(370): Chore service for: regionserver/1ff8009e463e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T12:46:41,588 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T12:46:41,588 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T12:46:41,588 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T12:46:41,588 INFO [RS:0;1ff8009e463e:43295 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:46:41,588 INFO [RS:0;1ff8009e463e:43295 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43295 2024-11-25T12:46:41,588 INFO [regionserver/1ff8009e463e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:46:41,593 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/WALs/1ff8009e463e,35205,1732538757028/1ff8009e463e%2C35205%2C1732538757028.1732538759942 not finished, retry = 0 2024-11-25T12:46:41,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741834_1010 (size=95) 2024-11-25T12:46:41,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741834_1010 (size=95) 2024-11-25T12:46:41,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:46:41,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ff8009e463e,43295,1732538756657 2024-11-25T12:46:41,599 INFO [RS:0;1ff8009e463e:43295 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:46:41,607 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ff8009e463e,43295,1732538756657] 2024-11-25T12:46:41,616 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ff8009e463e,43295,1732538756657 already deleted, retry=false 2024-11-25T12:46:41,616 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ff8009e463e,43295,1732538756657 expired; onlineServers=1 2024-11-25T12:46:41,696 DEBUG [RS:2;1ff8009e463e:35205 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/oldWALs 2024-11-25T12:46:41,696 INFO [RS:2;1ff8009e463e:35205 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ff8009e463e%2C35205%2C1732538757028:(num 1732538759942) 2024-11-25T12:46:41,696 DEBUG [RS:2;1ff8009e463e:35205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:46:41,696 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:46:41,697 INFO [RS:2;1ff8009e463e:35205 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:46:41,697 INFO [RS:2;1ff8009e463e:35205 {}] hbase.ChoreService(370): Chore service for: regionserver/1ff8009e463e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T12:46:41,697 INFO [RS:2;1ff8009e463e:35205 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:46:41,697 INFO [regionserver/1ff8009e463e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:46:41,697 INFO [RS:2;1ff8009e463e:35205 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35205 2024-11-25T12:46:41,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:46:41,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ff8009e463e,35205,1732538757028 2024-11-25T12:46:41,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:46:41,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43295-0x101735117c30001, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:46:41,708 INFO [RS:0;1ff8009e463e:43295 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:46:41,708 INFO [RS:0;1ff8009e463e:43295 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ff8009e463e,43295,1732538756657; zookeeper connection closed. 2024-11-25T12:46:41,708 INFO [RS:2;1ff8009e463e:35205 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:46:41,708 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a30b314 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a30b314 2024-11-25T12:46:41,708 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ff8009e463e,35205,1732538757028] 2024-11-25T12:46:41,724 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ff8009e463e,35205,1732538757028 already deleted, retry=false 2024-11-25T12:46:41,724 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ff8009e463e,35205,1732538757028 expired; onlineServers=0 2024-11-25T12:46:41,724 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1ff8009e463e,36927,1732538755691' ***** 2024-11-25T12:46:41,724 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T12:46:41,725 INFO [M:0;1ff8009e463e:36927 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:46:41,725 INFO [M:0;1ff8009e463e:36927 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:46:41,725 DEBUG [M:0;1ff8009e463e:36927 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T12:46:41,725 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T12:46:41,725 DEBUG [M:0;1ff8009e463e:36927 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T12:46:41,725 DEBUG [master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732538759297 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732538759297,5,FailOnTimeoutGroup] 2024-11-25T12:46:41,725 DEBUG [master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732538759299 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732538759299,5,FailOnTimeoutGroup] 2024-11-25T12:46:41,726 INFO [M:0;1ff8009e463e:36927 {}] hbase.ChoreService(370): Chore service for: master/1ff8009e463e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T12:46:41,726 INFO [M:0;1ff8009e463e:36927 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:46:41,726 DEBUG [M:0;1ff8009e463e:36927 {}] master.HMaster(1795): Stopping service threads 2024-11-25T12:46:41,726 INFO [M:0;1ff8009e463e:36927 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T12:46:41,727 INFO [M:0;1ff8009e463e:36927 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T12:46:41,728 INFO [M:0;1ff8009e463e:36927 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T12:46:41,728 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T12:46:41,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T12:46:41,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:46:41,736 DEBUG [M:0;1ff8009e463e:36927 {}] zookeeper.ZKUtil(347): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T12:46:41,737 WARN [M:0;1ff8009e463e:36927 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T12:46:41,737 INFO [M:0;1ff8009e463e:36927 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/.lastflushedseqids 2024-11-25T12:46:41,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741944_1124 (size=119) 2024-11-25T12:46:41,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741944_1124 (size=119) 2024-11-25T12:46:41,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741944_1124 (size=119) 2024-11-25T12:46:41,755 INFO [M:0;1ff8009e463e:36927 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T12:46:41,755 INFO [M:0;1ff8009e463e:36927 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T12:46:41,755 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T12:46:41,755 INFO [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:46:41,755 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:46:41,755 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T12:46:41,755 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:46:41,756 INFO [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-25T12:46:41,776 DEBUG [M:0;1ff8009e463e:36927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/24c7aef0bdb04330b1c914085e9e9eca is 82, key is hbase:meta,,1/info:regioninfo/1732538778512/Put/seqid=0 2024-11-25T12:46:41,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741945_1125 (size=6063) 2024-11-25T12:46:41,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741945_1125 (size=6063) 2024-11-25T12:46:41,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741945_1125 (size=6063) 2024-11-25T12:46:41,797 INFO [M:0;1ff8009e463e:36927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/24c7aef0bdb04330b1c914085e9e9eca 2024-11-25T12:46:41,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:46:41,816 INFO [RS:2;1ff8009e463e:35205 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:46:41,816 INFO [RS:2;1ff8009e463e:35205 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ff8009e463e,35205,1732538757028; zookeeper connection closed. 2024-11-25T12:46:41,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35205-0x101735117c30003, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:46:41,816 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@68ee856a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@68ee856a 2024-11-25T12:46:41,817 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-25T12:46:41,821 DEBUG [M:0;1ff8009e463e:36927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac7142331ea84690a0e08e695e2ab0cf is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732538775130/Put/seqid=0 2024-11-25T12:46:41,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741946_1126 (size=7907) 2024-11-25T12:46:41,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741946_1126 (size=7907) 2024-11-25T12:46:41,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741946_1126 (size=7907) 2024-11-25T12:46:41,831 INFO [M:0;1ff8009e463e:36927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac7142331ea84690a0e08e695e2ab0cf 2024-11-25T12:46:41,842 INFO [M:0;1ff8009e463e:36927 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ac7142331ea84690a0e08e695e2ab0cf 2024-11-25T12:46:41,865 DEBUG [M:0;1ff8009e463e:36927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be790f74b51142bf936436ac34e7de6c is 69, key is 1ff8009e463e,35205,1732538757028/rs:state/1732538759436/Put/seqid=0 2024-11-25T12:46:41,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741947_1127 (size=5445) 2024-11-25T12:46:41,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34959 is added to blk_1073741947_1127 (size=5445) 2024-11-25T12:46:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741947_1127 (size=5445) 2024-11-25T12:46:41,873 INFO [M:0;1ff8009e463e:36927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be790f74b51142bf936436ac34e7de6c 2024-11-25T12:46:41,878 INFO [M:0;1ff8009e463e:36927 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for be790f74b51142bf936436ac34e7de6c 2024-11-25T12:46:41,879 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/24c7aef0bdb04330b1c914085e9e9eca as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/24c7aef0bdb04330b1c914085e9e9eca 2024-11-25T12:46:41,884 INFO [M:0;1ff8009e463e:36927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/24c7aef0bdb04330b1c914085e9e9eca, entries=14, sequenceid=207, filesize=5.9 K 2024-11-25T12:46:41,885 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac7142331ea84690a0e08e695e2ab0cf as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ac7142331ea84690a0e08e695e2ab0cf 2024-11-25T12:46:41,893 INFO [M:0;1ff8009e463e:36927 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ac7142331ea84690a0e08e695e2ab0cf 2024-11-25T12:46:41,893 INFO [M:0;1ff8009e463e:36927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ac7142331ea84690a0e08e695e2ab0cf, entries=21, sequenceid=207, filesize=7.7 K 2024-11-25T12:46:41,895 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be790f74b51142bf936436ac34e7de6c as hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/be790f74b51142bf936436ac34e7de6c 2024-11-25T12:46:41,901 INFO [M:0;1ff8009e463e:36927 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for be790f74b51142bf936436ac34e7de6c 2024-11-25T12:46:41,901 INFO [M:0;1ff8009e463e:36927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35793/user/jenkins/test-data/95705c3e-54f8-6c8d-605a-c99a3e8458e7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/be790f74b51142bf936436ac34e7de6c, entries=3, sequenceid=207, filesize=5.3 K 2024-11-25T12:46:41,903 INFO [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=207, compaction requested=false 2024-11-25T12:46:41,913 INFO [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:46:41,913 DEBUG [M:0;1ff8009e463e:36927 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732538801755Disabling compacts and flushes for region at 1732538801755Disabling writes for close at 1732538801755Obtaining lock to block concurrent updates at 1732538801756 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732538801756Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1732538801756Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732538801757 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732538801757Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732538801776 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732538801776Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732538801802 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732538801820 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732538801820Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732538801842 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732538801864 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732538801864Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d268193: reopening flushed file at 1732538801878 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6740ab3a: reopening flushed file at 1732538801884 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21155813: reopening flushed file at 1732538801893 (+9 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=207, compaction requested=false at 1732538801903 (+10 ms)Writing region close event to WAL at 1732538801913 (+10 ms)Closed at 1732538801913 2024-11-25T12:46:41,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741830_1006 (size=69445) 2024-11-25T12:46:41,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33377 is added to blk_1073741830_1006 (size=69445) 2024-11-25T12:46:41,919 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:46:41,920 INFO [M:0;1ff8009e463e:36927 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T12:46:41,920 INFO [M:0;1ff8009e463e:36927 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36927 2024-11-25T12:46:41,922 INFO [M:0;1ff8009e463e:36927 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:46:42,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:46:42,033 INFO [M:0;1ff8009e463e:36927 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:46:42,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36927-0x101735117c30000, quorum=127.0.0.1:51952, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:46:42,043 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538790287 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538790287 (inode 16688) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538780806/wal.1732538790287 (inode 16688) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:46:42,051 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538780638 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538780638 (inode 16665) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538779303/wal.1732538780638 (inode 16665) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:46:42,052 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538769892/wal.1732538769994 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:42,059 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538791161 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538791161 (inode 16714) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538790588/wal.1732538791161 (inode 16714) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:46:42,060 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538791337/wal.1732538791402 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:42,060 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538796075/wal.1732538796427 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:42,060 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538762513/wal.1732538762630 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:46:42,067 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal.1732538762302 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal.1732538762302 (inode 16485) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538762028/wal.1732538762302 (inode 16485) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:46:42,071 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal.1732538761741 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal.1732538761741 (inode 16462) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538761113/wal.1732538761741 (inode 16462) Holder DFSClient_NONMAPREDUCE_-896067521_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:46:42,077 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26f8cb79{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:46:42,080 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:46:42,080 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:46:42,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:46:42,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir/,STOPPED} 2024-11-25T12:46:42,085 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T12:46:42,085 WARN [BP-225838851-172.17.0.3-1732538750914 heartbeating to localhost/127.0.0.1:35793 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T12:46:42,085 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T12:46:42,085 WARN [BP-225838851-172.17.0.3-1732538750914 heartbeating to localhost/127.0.0.1:35793 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-225838851-172.17.0.3-1732538750914 (Datanode Uuid 14b2755e-8265-4540-94ea-bca82d164c4d) service to localhost/127.0.0.1:35793 2024-11-25T12:46:42,087 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data5/current/BP-225838851-172.17.0.3-1732538750914 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:46:42,088 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data6/current/BP-225838851-172.17.0.3-1732538750914 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:46:42,091 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T12:46:42,096 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4464725f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:46:42,097 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:46:42,097 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:46:42,097 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:46:42,097 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir/,STOPPED} 2024-11-25T12:46:42,099 WARN [BP-225838851-172.17.0.3-1732538750914 heartbeating to localhost/127.0.0.1:35793 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T12:46:42,099 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T12:46:42,099 WARN [BP-225838851-172.17.0.3-1732538750914 heartbeating to localhost/127.0.0.1:35793 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-225838851-172.17.0.3-1732538750914 (Datanode Uuid d469b2b5-b085-42d7-aec4-85deee27acef) service to localhost/127.0.0.1:35793 2024-11-25T12:46:42,099 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T12:46:42,099 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data3/current/BP-225838851-172.17.0.3-1732538750914 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:46:42,100 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data4/current/BP-225838851-172.17.0.3-1732538750914 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:46:42,100 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T12:46:42,111 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@17d1a8d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:46:42,112 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:46:42,112 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:46:42,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:46:42,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir/,STOPPED} 2024-11-25T12:46:42,116 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T12:46:42,116 WARN [BP-225838851-172.17.0.3-1732538750914 heartbeating to localhost/127.0.0.1:35793 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T12:46:42,117 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T12:46:42,117 WARN [BP-225838851-172.17.0.3-1732538750914 heartbeating to localhost/127.0.0.1:35793 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-225838851-172.17.0.3-1732538750914 (Datanode Uuid d6dd8a94-3437-49ae-b57b-725cdaf69376) service to localhost/127.0.0.1:35793 2024-11-25T12:46:42,118 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data2/current/BP-225838851-172.17.0.3-1732538750914 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:46:42,118 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T12:46:42,119 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/cluster_87e50922-1b4f-5248-d9c0-149c809cf21a/data/data1/current/BP-225838851-172.17.0.3-1732538750914 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:46:42,126 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c8b857b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T12:46:42,127 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@58a7ff93{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:46:42,127 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:46:42,127 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a6f93cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:46:42,128 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dc5014{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/586e91fd-1a27-e462-8a16-32b9a5802243/hadoop.log.dir/,STOPPED} 2024-11-25T12:46:42,146 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T12:46:42,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down