2024-12-02 22:23:05,403 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-02 22:23:05,432 main DEBUG Took 0.024504 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 22:23:05,434 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 22:23:05,434 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 22:23:05,437 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 22:23:05,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,452 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 22:23:05,475 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,477 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,478 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,479 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,479 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,480 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,485 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,485 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,486 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,488 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,489 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,490 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,491 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,492 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,497 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,501 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,501 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:23:05,502 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,502 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 22:23:05,504 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:23:05,508 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 22:23:05,510 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 22:23:05,511 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 22:23:05,512 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 22:23:05,513 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 22:23:05,523 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 22:23:05,540 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 22:23:05,545 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 22:23:05,546 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 22:23:05,547 main DEBUG createAppenders(={Console}) 2024-12-02 22:23:05,548 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-02 22:23:05,549 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-02 22:23:05,550 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-02 22:23:05,551 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 22:23:05,552 main DEBUG OutputStream closed 2024-12-02 22:23:05,554 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 22:23:05,555 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 22:23:05,556 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-02 22:23:05,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 22:23:05,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 22:23:05,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 22:23:05,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 22:23:05,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 22:23:05,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 22:23:05,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 22:23:05,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 22:23:05,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 22:23:05,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 22:23:05,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 22:23:05,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 22:23:05,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 22:23:05,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 22:23:05,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 22:23:05,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 22:23:05,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 22:23:05,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 22:23:05,736 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 22:23:05,736 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-02 22:23:05,736 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 22:23:05,737 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-02T22:23:06,137 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911 2024-12-02 22:23:06,148 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 22:23:06,149 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T22:23:06,163 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-02T22:23:06,174 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-02T22:23:06,231 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T22:23:06,309 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T22:23:06,310 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T22:23:06,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T22:23:06,385 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274, deleteOnExit=true 2024-12-02T22:23:06,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T22:23:06,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/test.cache.data in system properties and HBase conf 2024-12-02T22:23:06,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T22:23:06,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir in system properties and HBase conf 2024-12-02T22:23:06,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T22:23:06,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T22:23:06,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T22:23:06,549 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T22:23:06,720 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T22:23:06,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T22:23:06,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T22:23:06,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T22:23:06,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T22:23:06,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T22:23:06,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T22:23:06,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T22:23:06,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T22:23:06,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T22:23:06,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/nfs.dump.dir in system properties and HBase conf 2024-12-02T22:23:06,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/java.io.tmpdir in system properties and HBase conf 2024-12-02T22:23:06,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T22:23:06,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T22:23:06,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T22:23:09,317 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T22:23:09,415 INFO [Time-limited test {}] log.Log(170): Logging initialized @5042ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T22:23:09,514 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T22:23:09,664 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T22:23:09,714 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T22:23:09,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T22:23:09,717 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T22:23:09,761 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T22:23:09,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a6ae9ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir/,AVAILABLE} 2024-12-02T22:23:09,772 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@120a4c3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T22:23:10,062 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14f32243{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/java.io.tmpdir/jetty-localhost-35153-hadoop-hdfs-3_4_1-tests_jar-_-any-16383946537163582891/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T22:23:10,072 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71a2a7f2{HTTP/1.1, (http/1.1)}{localhost:35153} 2024-12-02T22:23:10,073 INFO [Time-limited test {}] server.Server(415): Started @5700ms 2024-12-02T22:23:11,293 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T22:23:11,302 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T22:23:11,305 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T22:23:11,305 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T22:23:11,305 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T22:23:11,306 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49d86915{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir/,AVAILABLE} 2024-12-02T22:23:11,307 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ace2437{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T22:23:11,471 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d480c14{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/java.io.tmpdir/jetty-localhost-44741-hadoop-hdfs-3_4_1-tests_jar-_-any-13442894350133989078/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T22:23:11,472 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8a1e1ee{HTTP/1.1, (http/1.1)}{localhost:44741} 2024-12-02T22:23:11,472 INFO [Time-limited test {}] server.Server(415): Started @7099ms 2024-12-02T22:23:11,544 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T22:23:11,764 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T22:23:11,783 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T22:23:11,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T22:23:11,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T22:23:11,789 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T22:23:11,796 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78a777d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir/,AVAILABLE} 2024-12-02T22:23:11,797 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@14fd625f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T22:23:11,999 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75d96791{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/java.io.tmpdir/jetty-localhost-34611-hadoop-hdfs-3_4_1-tests_jar-_-any-4788426151725896549/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T22:23:12,000 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@60aef58b{HTTP/1.1, (http/1.1)}{localhost:34611} 2024-12-02T22:23:12,000 INFO [Time-limited test {}] server.Server(415): Started @7627ms 2024-12-02T22:23:12,003 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T22:23:12,078 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T22:23:12,087 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T22:23:12,102 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T22:23:12,102 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T22:23:12,102 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T22:23:12,105 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78fad685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir/,AVAILABLE} 2024-12-02T22:23:12,106 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b0de5e3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T22:23:12,238 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@773b4a40{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/java.io.tmpdir/jetty-localhost-45227-hadoop-hdfs-3_4_1-tests_jar-_-any-6520405527923435969/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T22:23:12,238 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@286fa8e3{HTTP/1.1, (http/1.1)}{localhost:45227} 2024-12-02T22:23:12,239 INFO [Time-limited test {}] server.Server(415): Started @7865ms 2024-12-02T22:23:12,241 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T22:23:13,981 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data2/current/BP-981501226-172.17.0.2-1733178187608/current, will proceed with Du for space computation calculation, 2024-12-02T22:23:13,981 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data1/current/BP-981501226-172.17.0.2-1733178187608/current, will proceed with Du for space computation calculation, 2024-12-02T22:23:14,017 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T22:23:14,018 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data3/current/BP-981501226-172.17.0.2-1733178187608/current, will proceed with Du for space computation calculation, 2024-12-02T22:23:14,018 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data4/current/BP-981501226-172.17.0.2-1733178187608/current, will proceed with Du for space computation calculation, 2024-12-02T22:23:14,047 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T22:23:14,080 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc30a9f06e5172234 with lease ID 0x5f6016be4ab35903: Processing first storage report for DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c from datanode DatanodeRegistration(127.0.0.1:34817, datanodeUuid=672d9a78-cfc9-4905-bee0-1d905203b49b, infoPort=45007, infoSecurePort=0, ipcPort=43187, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608) 2024-12-02T22:23:14,081 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc30a9f06e5172234 with lease ID 0x5f6016be4ab35903: from storage DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c node DatanodeRegistration(127.0.0.1:34817, datanodeUuid=672d9a78-cfc9-4905-bee0-1d905203b49b, infoPort=45007, infoSecurePort=0, ipcPort=43187, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T22:23:14,081 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x805500373cdc2ad7 with lease ID 0x5f6016be4ab35904: Processing first storage report for DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd from datanode DatanodeRegistration(127.0.0.1:40725, datanodeUuid=bce2a7ab-d5be-4824-bd59-025c5634e40b, infoPort=36155, infoSecurePort=0, ipcPort=38099, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608) 2024-12-02T22:23:14,082 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x805500373cdc2ad7 with lease ID 0x5f6016be4ab35904: from storage DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd node DatanodeRegistration(127.0.0.1:40725, datanodeUuid=bce2a7ab-d5be-4824-bd59-025c5634e40b, infoPort=36155, infoSecurePort=0, ipcPort=38099, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T22:23:14,082 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc30a9f06e5172234 with lease ID 0x5f6016be4ab35903: Processing first storage report for DS-1821655b-1746-4b06-ab94-4e382f23052f from datanode DatanodeRegistration(127.0.0.1:34817, datanodeUuid=672d9a78-cfc9-4905-bee0-1d905203b49b, infoPort=45007, infoSecurePort=0, ipcPort=43187, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608) 2024-12-02T22:23:14,082 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc30a9f06e5172234 with lease ID 0x5f6016be4ab35903: from storage DS-1821655b-1746-4b06-ab94-4e382f23052f node DatanodeRegistration(127.0.0.1:34817, datanodeUuid=672d9a78-cfc9-4905-bee0-1d905203b49b, infoPort=45007, infoSecurePort=0, ipcPort=43187, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T22:23:14,082 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x805500373cdc2ad7 with lease ID 0x5f6016be4ab35904: Processing first storage report for DS-92466606-8283-476a-bbfb-6b8ed215ad6b from datanode DatanodeRegistration(127.0.0.1:40725, datanodeUuid=bce2a7ab-d5be-4824-bd59-025c5634e40b, infoPort=36155, infoSecurePort=0, ipcPort=38099, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608) 2024-12-02T22:23:14,082 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x805500373cdc2ad7 with lease ID 0x5f6016be4ab35904: from storage DS-92466606-8283-476a-bbfb-6b8ed215ad6b node DatanodeRegistration(127.0.0.1:40725, datanodeUuid=bce2a7ab-d5be-4824-bd59-025c5634e40b, infoPort=36155, infoSecurePort=0, ipcPort=38099, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T22:23:14,141 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data6/current/BP-981501226-172.17.0.2-1733178187608/current, will proceed with Du for space computation calculation, 2024-12-02T22:23:14,141 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data5/current/BP-981501226-172.17.0.2-1733178187608/current, will proceed with Du for space computation calculation, 2024-12-02T22:23:14,165 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T22:23:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd5a1ef6ef9686ded with lease ID 0x5f6016be4ab35905: Processing first storage report for DS-57e02b84-9b94-4395-a6ba-37fe566993d5 from datanode DatanodeRegistration(127.0.0.1:37315, datanodeUuid=60203b4e-7dac-4b99-80de-eb0a6d1e92d9, infoPort=37917, infoSecurePort=0, ipcPort=35905, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608) 2024-12-02T22:23:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd5a1ef6ef9686ded with lease ID 0x5f6016be4ab35905: from storage DS-57e02b84-9b94-4395-a6ba-37fe566993d5 node DatanodeRegistration(127.0.0.1:37315, datanodeUuid=60203b4e-7dac-4b99-80de-eb0a6d1e92d9, infoPort=37917, infoSecurePort=0, ipcPort=35905, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T22:23:14,171 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd5a1ef6ef9686ded with lease ID 0x5f6016be4ab35905: Processing first storage report for DS-ba8c72ab-1090-4cf8-bf57-e87742afbc85 from datanode DatanodeRegistration(127.0.0.1:37315, datanodeUuid=60203b4e-7dac-4b99-80de-eb0a6d1e92d9, infoPort=37917, infoSecurePort=0, ipcPort=35905, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608) 2024-12-02T22:23:14,171 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd5a1ef6ef9686ded with lease ID 0x5f6016be4ab35905: from storage DS-ba8c72ab-1090-4cf8-bf57-e87742afbc85 node DatanodeRegistration(127.0.0.1:37315, datanodeUuid=60203b4e-7dac-4b99-80de-eb0a6d1e92d9, infoPort=37917, infoSecurePort=0, ipcPort=35905, storageInfo=lv=-57;cid=testClusterID;nsid=463892338;c=1733178187608), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T22:23:14,205 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911 2024-12-02T22:23:14,341 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/zookeeper_0, clientPort=55591, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T22:23:14,351 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55591 2024-12-02T22:23:14,368 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:14,371 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:14,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741825_1001 (size=7) 2024-12-02T22:23:14,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741825_1001 (size=7) 2024-12-02T22:23:14,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741825_1001 (size=7) 2024-12-02T22:23:15,029 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3 with version=8 2024-12-02T22:23:15,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/hbase-staging 2024-12-02T22:23:15,411 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7ae73638d47e:0 server-side Connection retries=45 2024-12-02T22:23:15,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:15,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:15,431 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T22:23:15,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:15,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T22:23:15,613 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T22:23:15,689 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T22:23:15,698 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T22:23:15,703 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T22:23:15,734 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2662 (auto-detected) 2024-12-02T22:23:15,735 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-02T22:23:15,757 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42475 2024-12-02T22:23:15,781 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42475 connecting to ZooKeeper ensemble=127.0.0.1:55591 2024-12-02T22:23:15,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:424750x0, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T22:23:15,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42475-0x1009d16841a0000 connected 2024-12-02T22:23:16,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,051 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,067 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:23:16,071 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3, hbase.cluster.distributed=false 2024-12-02T22:23:16,099 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T22:23:16,105 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42475 2024-12-02T22:23:16,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42475 2024-12-02T22:23:16,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42475 2024-12-02T22:23:16,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42475 2024-12-02T22:23:16,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42475 2024-12-02T22:23:16,234 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7ae73638d47e:0 server-side Connection retries=45 2024-12-02T22:23:16,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,237 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T22:23:16,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T22:23:16,240 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T22:23:16,242 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T22:23:16,243 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38411 2024-12-02T22:23:16,245 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38411 connecting to ZooKeeper ensemble=127.0.0.1:55591 2024-12-02T22:23:16,246 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,250 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384110x0, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T22:23:16,283 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38411-0x1009d16841a0001 connected 2024-12-02T22:23:16,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:23:16,289 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T22:23:16,306 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T22:23:16,309 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T22:23:16,315 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T22:23:16,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38411 2024-12-02T22:23:16,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38411 2024-12-02T22:23:16,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38411 2024-12-02T22:23:16,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38411 2024-12-02T22:23:16,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38411 2024-12-02T22:23:16,346 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7ae73638d47e:0 server-side Connection retries=45 2024-12-02T22:23:16,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,347 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T22:23:16,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T22:23:16,347 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T22:23:16,348 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T22:23:16,349 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35511 2024-12-02T22:23:16,350 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35511 connecting to ZooKeeper ensemble=127.0.0.1:55591 2024-12-02T22:23:16,351 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,353 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355110x0, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T22:23:16,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:355110x0, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:23:16,378 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T22:23:16,385 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35511-0x1009d16841a0002 connected 2024-12-02T22:23:16,386 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T22:23:16,387 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T22:23:16,389 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T22:23:16,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35511 2024-12-02T22:23:16,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35511 2024-12-02T22:23:16,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35511 2024-12-02T22:23:16,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35511 2024-12-02T22:23:16,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35511 2024-12-02T22:23:16,419 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7ae73638d47e:0 server-side Connection retries=45 2024-12-02T22:23:16,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,420 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T22:23:16,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:23:16,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T22:23:16,421 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T22:23:16,421 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T22:23:16,422 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45899 2024-12-02T22:23:16,424 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45899 connecting to ZooKeeper ensemble=127.0.0.1:55591 2024-12-02T22:23:16,425 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458990x0, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T22:23:16,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45899-0x1009d16841a0003 connected 2024-12-02T22:23:16,453 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:23:16,454 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T22:23:16,455 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T22:23:16,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T22:23:16,458 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T22:23:16,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45899 2024-12-02T22:23:16,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45899 2024-12-02T22:23:16,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45899 2024-12-02T22:23:16,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45899 2024-12-02T22:23:16,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45899 2024-12-02T22:23:16,478 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7ae73638d47e:42475 2024-12-02T22:23:16,479 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7ae73638d47e,42475,1733178195164 2024-12-02T22:23:16,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:23:16,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:23:16,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:23:16,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:23:16,501 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7ae73638d47e,42475,1733178195164 2024-12-02T22:23:16,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T22:23:16,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T22:23:16,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:16,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T22:23:16,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:16,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:16,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:16,561 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T22:23:16,563 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7ae73638d47e,42475,1733178195164 from backup master directory 2024-12-02T22:23:16,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:23:16,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:23:16,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:23:16,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7ae73638d47e,42475,1733178195164 2024-12-02T22:23:16,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:23:16,600 WARN [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T22:23:16,600 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7ae73638d47e,42475,1733178195164 2024-12-02T22:23:16,602 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T22:23:16,604 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T22:23:16,678 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/hbase.id] with ID: 9163252c-8e97-41ba-b6b3-4a5e364dbbc9 2024-12-02T22:23:16,678 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/.tmp/hbase.id 2024-12-02T22:23:16,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741826_1002 (size=42) 2024-12-02T22:23:16,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741826_1002 (size=42) 2024-12-02T22:23:16,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741826_1002 (size=42) 2024-12-02T22:23:16,714 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/.tmp/hbase.id]:[hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/hbase.id] 2024-12-02T22:23:16,781 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:23:16,787 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T22:23:16,810 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-02T22:23:16,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:16,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:16,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:16,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:16,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741827_1003 (size=196) 2024-12-02T22:23:16,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741827_1003 (size=196) 2024-12-02T22:23:16,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741827_1003 (size=196) 2024-12-02T22:23:16,886 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:23:16,888 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T22:23:16,895 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:16,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741828_1004 (size=1189) 2024-12-02T22:23:16,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741828_1004 (size=1189) 2024-12-02T22:23:16,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741828_1004 (size=1189) 2024-12-02T22:23:16,973 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store 2024-12-02T22:23:17,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741829_1005 (size=34) 2024-12-02T22:23:17,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741829_1005 (size=34) 2024-12-02T22:23:17,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741829_1005 (size=34) 2024-12-02T22:23:17,008 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T22:23:17,016 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:17,017 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T22:23:17,017 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:23:17,018 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:23:17,019 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T22:23:17,020 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:23:17,020 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:23:17,021 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733178197017Disabling compacts and flushes for region at 1733178197017Disabling writes for close at 1733178197019 (+2 ms)Writing region close event to WAL at 1733178197020 (+1 ms)Closed at 1733178197020 2024-12-02T22:23:17,024 WARN [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/.initializing 2024-12-02T22:23:17,024 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/WALs/7ae73638d47e,42475,1733178195164 2024-12-02T22:23:17,036 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:17,058 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C42475%2C1733178195164, suffix=, logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/WALs/7ae73638d47e,42475,1733178195164, archiveDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/oldWALs, maxLogs=10 2024-12-02T22:23:17,102 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/WALs/7ae73638d47e,42475,1733178195164/7ae73638d47e%2C42475%2C1733178195164.1733178197064, exclude list is [], retry=0 2024-12-02T22:23:17,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:17,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:17,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:17,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:17,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T22:23:17,182 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-02T22:23:17,276 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/WALs/7ae73638d47e,42475,1733178195164/7ae73638d47e%2C42475%2C1733178195164.1733178197064 2024-12-02T22:23:17,277 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:23:17,278 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:17,278 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:17,284 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,285 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T22:23:17,400 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:17,403 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:17,404 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,410 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T22:23:17,410 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:17,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:17,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,421 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T22:23:17,421 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:17,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:17,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T22:23:17,429 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:17,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:17,431 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,435 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,436 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,441 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,442 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,446 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T22:23:17,450 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:23:17,456 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:17,458 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68442324, jitterRate=0.019870102405548096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T22:23:17,466 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733178197303Initializing all the Stores at 1733178197307 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178197308 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178197313 (+5 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178197313Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178197314 (+1 ms)Cleaning up temporary data from old regions at 1733178197442 (+128 ms)Region opened successfully at 1733178197466 (+24 ms) 2024-12-02T22:23:17,467 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T22:23:17,513 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef78051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7ae73638d47e/172.17.0.2:0 2024-12-02T22:23:17,566 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T22:23:17,586 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T22:23:17,586 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T22:23:17,589 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T22:23:17,597 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 7 msec 2024-12-02T22:23:17,618 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 20 msec 2024-12-02T22:23:17,618 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T22:23:17,685 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T22:23:17,697 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T22:23:17,755 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T22:23:17,759 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T22:23:17,771 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T22:23:17,860 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T22:23:17,863 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T22:23:17,868 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T22:23:17,918 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T22:23:17,924 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T22:23:17,960 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T22:23:18,007 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T22:23:18,044 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T22:23:18,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T22:23:18,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T22:23:18,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T22:23:18,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T22:23:18,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,084 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7ae73638d47e,42475,1733178195164, sessionid=0x1009d16841a0000, setting cluster-up flag (Was=false) 2024-12-02T22:23:18,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,410 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T22:23:18,426 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7ae73638d47e,42475,1733178195164 2024-12-02T22:23:18,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:18,630 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T22:23:18,654 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7ae73638d47e,42475,1733178195164 2024-12-02T22:23:18,679 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T22:23:18,790 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(746): ClusterId : 9163252c-8e97-41ba-b6b3-4a5e364dbbc9 2024-12-02T22:23:18,806 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T22:23:18,808 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(746): ClusterId : 9163252c-8e97-41ba-b6b3-4a5e364dbbc9 2024-12-02T22:23:18,808 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T22:23:18,817 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(746): ClusterId : 9163252c-8e97-41ba-b6b3-4a5e364dbbc9 2024-12-02T22:23:18,817 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T22:23:18,875 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T22:23:18,875 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T22:23:18,875 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T22:23:18,875 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T22:23:18,891 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T22:23:18,910 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T22:23:18,921 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T22:23:18,921 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T22:23:18,925 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T22:23:18,926 DEBUG [RS:1;7ae73638d47e:35511 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e09ea93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7ae73638d47e/172.17.0.2:0 2024-12-02T22:23:18,929 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T22:23:18,938 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7ae73638d47e,42475,1733178195164 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T22:23:18,961 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T22:23:18,962 DEBUG [RS:0;7ae73638d47e:38411 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244ac048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7ae73638d47e/172.17.0.2:0 2024-12-02T22:23:18,964 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T22:23:18,965 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7ae73638d47e:0, corePoolSize=5, maxPoolSize=5 2024-12-02T22:23:18,965 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7ae73638d47e:0, corePoolSize=5, maxPoolSize=5 2024-12-02T22:23:18,965 DEBUG [RS:2;7ae73638d47e:45899 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fca3b5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7ae73638d47e/172.17.0.2:0 2024-12-02T22:23:18,965 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7ae73638d47e:0, corePoolSize=5, maxPoolSize=5 2024-12-02T22:23:18,965 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7ae73638d47e:0, corePoolSize=5, maxPoolSize=5 2024-12-02T22:23:18,966 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7ae73638d47e:0, corePoolSize=10, maxPoolSize=10 2024-12-02T22:23:18,966 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:18,966 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7ae73638d47e:0, corePoolSize=2, maxPoolSize=2 2024-12-02T22:23:18,966 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:18,976 DEBUG [RS:1;7ae73638d47e:35511 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7ae73638d47e:35511 2024-12-02T22:23:18,987 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T22:23:18,990 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T22:23:18,992 DEBUG [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T22:23:18,999 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(2659): reportForDuty to master=7ae73638d47e,42475,1733178195164 with port=35511, startcode=1733178196345 2024-12-02T22:23:19,012 DEBUG [RS:0;7ae73638d47e:38411 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7ae73638d47e:38411 2024-12-02T22:23:19,012 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T22:23:19,012 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T22:23:19,012 DEBUG [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T22:23:19,014 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T22:23:19,014 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T22:23:19,015 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733178229015 2024-12-02T22:23:19,017 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T22:23:19,018 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T22:23:19,023 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T22:23:19,023 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T22:23:19,024 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T22:23:19,024 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T22:23:19,025 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(2659): reportForDuty to master=7ae73638d47e,42475,1733178195164 with port=38411, startcode=1733178196190 2024-12-02T22:23:19,026 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,038 DEBUG [RS:2;7ae73638d47e:45899 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7ae73638d47e:45899 2024-12-02T22:23:19,038 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T22:23:19,038 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T22:23:19,038 DEBUG [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T22:23:19,041 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(2659): reportForDuty to master=7ae73638d47e,42475,1733178195164 with port=45899, startcode=1733178196419 2024-12-02T22:23:19,043 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:19,043 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T22:23:19,046 DEBUG [RS:1;7ae73638d47e:35511 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T22:23:19,046 DEBUG [RS:0;7ae73638d47e:38411 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T22:23:19,047 DEBUG [RS:2;7ae73638d47e:45899 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T22:23:19,055 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T22:23:19,057 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T22:23:19,057 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T22:23:19,061 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T22:23:19,061 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T22:23:19,072 WARN [IPC Server handler 1 on default port 38075 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T22:23:19,073 WARN [IPC Server handler 1 on default port 38075 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T22:23:19,073 WARN [IPC Server handler 1 on default port 38075 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T22:23:19,085 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.large.0-1733178199063,5,FailOnTimeoutGroup] 2024-12-02T22:23:19,092 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.small.0-1733178199086,5,FailOnTimeoutGroup] 2024-12-02T22:23:19,093 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,093 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T22:23:19,094 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,095 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741831_1007 (size=1321) 2024-12-02T22:23:19,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741831_1007 (size=1321) 2024-12-02T22:23:19,132 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T22:23:19,132 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3 2024-12-02T22:23:19,182 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38149, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T22:23:19,186 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54345, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T22:23:19,187 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57823, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T22:23:19,193 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7ae73638d47e,35511,1733178196345 2024-12-02T22:23:19,196 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(517): Registering regionserver=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:19,222 DEBUG [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3 2024-12-02T22:23:19,222 DEBUG [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38075 2024-12-02T22:23:19,223 DEBUG [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T22:23:19,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741832_1008 (size=32) 2024-12-02T22:23:19,254 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7ae73638d47e,38411,1733178196190 2024-12-02T22:23:19,255 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(517): Registering regionserver=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:19,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741832_1008 (size=32) 2024-12-02T22:23:19,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741832_1008 (size=32) 2024-12-02T22:23:19,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:19,265 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7ae73638d47e,45899,1733178196419 2024-12-02T22:23:19,265 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(517): Registering regionserver=7ae73638d47e,45899,1733178196419 2024-12-02T22:23:19,268 DEBUG [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3 2024-12-02T22:23:19,268 DEBUG [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38075 2024-12-02T22:23:19,269 DEBUG [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T22:23:19,271 DEBUG [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3 2024-12-02T22:23:19,271 DEBUG [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38075 2024-12-02T22:23:19,272 DEBUG [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T22:23:19,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T22:23:19,296 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T22:23:19,296 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:19,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:19,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T22:23:19,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T22:23:19,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T22:23:19,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:19,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:19,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T22:23:19,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T22:23:19,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:19,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:19,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T22:23:19,385 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T22:23:19,385 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:19,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:19,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T22:23:19,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740 2024-12-02T22:23:19,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740 2024-12-02T22:23:19,395 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T22:23:19,395 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T22:23:19,396 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T22:23:19,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T22:23:19,414 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:19,415 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70511514, jitterRate=0.05070343613624573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T22:23:19,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733178199262Initializing all the Stores at 1733178199266 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178199266Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178199288 (+22 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178199289 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178199289Cleaning up temporary data from old regions at 1733178199395 (+106 ms)Region opened successfully at 1733178199419 (+24 ms) 2024-12-02T22:23:19,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T22:23:19,419 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T22:23:19,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T22:23:19,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T22:23:19,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T22:23:19,429 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T22:23:19,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733178199419Disabling compacts and flushes for region at 1733178199419Disabling writes for close at 1733178199420 (+1 ms)Writing region close event to WAL at 1733178199428 (+8 ms)Closed at 1733178199429 (+1 ms) 2024-12-02T22:23:19,434 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T22:23:19,434 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T22:23:19,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T22:23:19,470 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T22:23:19,479 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T22:23:19,491 DEBUG [RS:1;7ae73638d47e:35511 {}] zookeeper.ZKUtil(111): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7ae73638d47e,35511,1733178196345 2024-12-02T22:23:19,491 WARN [RS:1;7ae73638d47e:35511 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T22:23:19,491 INFO [RS:1;7ae73638d47e:35511 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:19,491 DEBUG [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,35511,1733178196345 2024-12-02T22:23:19,495 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7ae73638d47e,38411,1733178196190] 2024-12-02T22:23:19,495 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7ae73638d47e,35511,1733178196345] 2024-12-02T22:23:19,495 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7ae73638d47e,45899,1733178196419] 2024-12-02T22:23:19,496 DEBUG [RS:0;7ae73638d47e:38411 {}] zookeeper.ZKUtil(111): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7ae73638d47e,38411,1733178196190 2024-12-02T22:23:19,496 DEBUG [RS:2;7ae73638d47e:45899 {}] zookeeper.ZKUtil(111): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7ae73638d47e,45899,1733178196419 2024-12-02T22:23:19,496 WARN [RS:0;7ae73638d47e:38411 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T22:23:19,496 WARN [RS:2;7ae73638d47e:45899 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T22:23:19,496 INFO [RS:2;7ae73638d47e:45899 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:19,496 INFO [RS:0;7ae73638d47e:38411 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:19,496 DEBUG [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,45899,1733178196419 2024-12-02T22:23:19,496 DEBUG [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190 2024-12-02T22:23:19,541 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T22:23:19,574 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T22:23:19,583 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T22:23:19,607 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T22:23:19,612 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T22:23:19,616 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T22:23:19,618 INFO [RS:0;7ae73638d47e:38411 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T22:23:19,618 INFO [RS:1;7ae73638d47e:35511 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T22:23:19,618 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,618 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,624 INFO [RS:2;7ae73638d47e:45899 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T22:23:19,624 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,632 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T22:23:19,633 WARN [7ae73638d47e:42475 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T22:23:19,633 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T22:23:19,639 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T22:23:19,642 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T22:23:19,644 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,644 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,644 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,645 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,645 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,645 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,645 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0, corePoolSize=2, maxPoolSize=2 2024-12-02T22:23:19,645 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,645 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,646 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,646 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,646 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,646 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T22:23:19,646 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,646 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7ae73638d47e:0, corePoolSize=3, maxPoolSize=3 2024-12-02T22:23:19,646 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,646 DEBUG [RS:0;7ae73638d47e:38411 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0, corePoolSize=3, maxPoolSize=3 2024-12-02T22:23:19,646 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,647 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,647 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,647 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,647 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,647 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0, corePoolSize=2, maxPoolSize=2 2024-12-02T22:23:19,647 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,648 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,648 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,648 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,648 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,648 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,648 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7ae73638d47e:0, corePoolSize=3, maxPoolSize=3 2024-12-02T22:23:19,648 DEBUG [RS:2;7ae73638d47e:45899 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0, corePoolSize=3, maxPoolSize=3 2024-12-02T22:23:19,651 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T22:23:19,651 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,652 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,652 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,652 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,652 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,652 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,652 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0, corePoolSize=2, maxPoolSize=2 2024-12-02T22:23:19,652 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,653 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,653 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,653 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,653 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,653 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:23:19,653 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7ae73638d47e:0, corePoolSize=3, maxPoolSize=3 2024-12-02T22:23:19,653 DEBUG [RS:1;7ae73638d47e:35511 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0, corePoolSize=3, maxPoolSize=3 2024-12-02T22:23:19,688 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,689 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,689 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,689 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,689 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,689 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,45899,1733178196419-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T22:23:19,707 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,707 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,707 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,708 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,708 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,708 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,38411,1733178196190-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T22:23:19,728 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,729 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,729 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,729 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,729 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,729 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,35511,1733178196345-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T22:23:19,773 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T22:23:19,778 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,38411,1733178196190-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,780 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,784 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.Replication(171): 7ae73638d47e,38411,1733178196190 started 2024-12-02T22:23:19,798 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T22:23:19,800 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T22:23:19,808 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,35511,1733178196345-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,809 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,809 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.Replication(171): 7ae73638d47e,35511,1733178196345 started 2024-12-02T22:23:19,813 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,45899,1733178196419-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,814 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,814 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.Replication(171): 7ae73638d47e,45899,1733178196419 started 2024-12-02T22:23:19,882 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,898 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1482): Serving as 7ae73638d47e,38411,1733178196190, RpcServer on 7ae73638d47e/172.17.0.2:38411, sessionid=0x1009d16841a0001 2024-12-02T22:23:19,899 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T22:23:19,899 DEBUG [RS:0;7ae73638d47e:38411 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7ae73638d47e,38411,1733178196190 2024-12-02T22:23:19,899 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ae73638d47e,38411,1733178196190' 2024-12-02T22:23:19,899 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T22:23:19,904 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T22:23:19,906 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T22:23:19,906 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T22:23:19,906 DEBUG [RS:0;7ae73638d47e:38411 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7ae73638d47e,38411,1733178196190 2024-12-02T22:23:19,906 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ae73638d47e,38411,1733178196190' 2024-12-02T22:23:19,906 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T22:23:19,907 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,908 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T22:23:19,908 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(1482): Serving as 7ae73638d47e,45899,1733178196419, RpcServer on 7ae73638d47e/172.17.0.2:45899, sessionid=0x1009d16841a0003 2024-12-02T22:23:19,909 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T22:23:19,909 DEBUG [RS:2;7ae73638d47e:45899 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7ae73638d47e,45899,1733178196419 2024-12-02T22:23:19,909 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ae73638d47e,45899,1733178196419' 2024-12-02T22:23:19,909 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T22:23:19,910 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:19,910 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1482): Serving as 7ae73638d47e,35511,1733178196345, RpcServer on 7ae73638d47e/172.17.0.2:35511, sessionid=0x1009d16841a0002 2024-12-02T22:23:19,911 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T22:23:19,911 DEBUG [RS:1;7ae73638d47e:35511 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7ae73638d47e,35511,1733178196345 2024-12-02T22:23:19,911 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ae73638d47e,35511,1733178196345' 2024-12-02T22:23:19,911 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T22:23:19,913 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T22:23:19,914 DEBUG [RS:0;7ae73638d47e:38411 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T22:23:19,914 INFO [RS:0;7ae73638d47e:38411 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T22:23:19,914 INFO [RS:0;7ae73638d47e:38411 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T22:23:19,915 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T22:23:19,915 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T22:23:19,915 DEBUG [RS:2;7ae73638d47e:45899 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7ae73638d47e,45899,1733178196419 2024-12-02T22:23:19,915 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ae73638d47e,45899,1733178196419' 2024-12-02T22:23:19,915 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T22:23:19,916 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T22:23:19,916 DEBUG [RS:2;7ae73638d47e:45899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T22:23:19,916 INFO [RS:2;7ae73638d47e:45899 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T22:23:19,917 INFO [RS:2;7ae73638d47e:45899 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T22:23:19,919 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T22:23:19,920 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T22:23:19,920 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T22:23:19,920 DEBUG [RS:1;7ae73638d47e:35511 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7ae73638d47e,35511,1733178196345 2024-12-02T22:23:19,920 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ae73638d47e,35511,1733178196345' 2024-12-02T22:23:19,920 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T22:23:19,921 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T22:23:19,922 DEBUG [RS:1;7ae73638d47e:35511 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T22:23:19,922 INFO [RS:1;7ae73638d47e:35511 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T22:23:19,922 INFO [RS:1;7ae73638d47e:35511 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T22:23:20,020 INFO [RS:2;7ae73638d47e:45899 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:20,020 INFO [RS:0;7ae73638d47e:38411 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:20,023 INFO [RS:1;7ae73638d47e:35511 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:20,023 INFO [RS:0;7ae73638d47e:38411 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C38411%2C1733178196190, suffix=, logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190, archiveDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs, maxLogs=32 2024-12-02T22:23:20,023 INFO [RS:2;7ae73638d47e:45899 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C45899%2C1733178196419, suffix=, logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,45899,1733178196419, archiveDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs, maxLogs=32 2024-12-02T22:23:20,027 INFO [RS:1;7ae73638d47e:35511 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C35511%2C1733178196345, suffix=, logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,35511,1733178196345, archiveDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs, maxLogs=32 2024-12-02T22:23:20,047 DEBUG [RS:2;7ae73638d47e:45899 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,45899,1733178196419/7ae73638d47e%2C45899%2C1733178196419.1733178200028, exclude list is [], retry=0 2024-12-02T22:23:20,050 WARN [IPC Server handler 2 on default port 38075 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T22:23:20,050 WARN [IPC Server handler 2 on default port 38075 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T22:23:20,050 DEBUG [RS:0;7ae73638d47e:38411 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190/7ae73638d47e%2C38411%2C1733178196190.1733178200029, exclude list is [], retry=0 2024-12-02T22:23:20,050 DEBUG [RS:1;7ae73638d47e:35511 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,35511,1733178196345/7ae73638d47e%2C35511%2C1733178196345.1733178200029, exclude list is [], retry=0 2024-12-02T22:23:20,052 WARN [IPC Server handler 2 on default port 38075 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T22:23:20,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:20,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:20,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:20,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:20,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:20,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:20,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:20,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:20,141 INFO [RS:2;7ae73638d47e:45899 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,45899,1733178196419/7ae73638d47e%2C45899%2C1733178196419.1733178200028 2024-12-02T22:23:20,143 DEBUG [RS:2;7ae73638d47e:45899 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:23:20,193 INFO [RS:0;7ae73638d47e:38411 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190/7ae73638d47e%2C38411%2C1733178196190.1733178200029 2024-12-02T22:23:20,195 DEBUG [RS:0;7ae73638d47e:38411 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:23:20,223 INFO [RS:1;7ae73638d47e:35511 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,35511,1733178196345/7ae73638d47e%2C35511%2C1733178196345.1733178200029 2024-12-02T22:23:20,224 DEBUG [RS:1;7ae73638d47e:35511 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:23:20,386 DEBUG [7ae73638d47e:42475 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-02T22:23:20,401 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(204): Hosts are {7ae73638d47e=0} racks are {/default-rack=0} 2024-12-02T22:23:20,424 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T22:23:20,424 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T22:23:20,424 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T22:23:20,424 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T22:23:20,424 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T22:23:20,424 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T22:23:20,424 INFO [7ae73638d47e:42475 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T22:23:20,424 INFO [7ae73638d47e:42475 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T22:23:20,424 INFO [7ae73638d47e:42475 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T22:23:20,424 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T22:23:20,435 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:20,441 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7ae73638d47e,38411,1733178196190, state=OPENING 2024-12-02T22:23:20,560 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T22:23:20,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:20,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:20,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:20,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:23:20,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:20,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:20,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:20,655 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T22:23:20,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:20,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7ae73638d47e,38411,1733178196190}] 2024-12-02T22:23:20,846 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T22:23:20,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55323, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T22:23:20,875 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T22:23:20,876 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:20,878 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T22:23:20,888 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C38411%2C1733178196190.meta, suffix=.meta, logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190, archiveDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs, maxLogs=32 2024-12-02T22:23:20,919 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, exclude list is [], retry=0 2024-12-02T22:23:20,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:20,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:20,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:21,075 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta 2024-12-02T22:23:21,076 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:23:21,077 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:21,082 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T22:23:21,085 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T22:23:21,104 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T22:23:21,111 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T22:23:21,112 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:21,112 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T22:23:21,112 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T22:23:21,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T22:23:21,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T22:23:21,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:21,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:21,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T22:23:21,124 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T22:23:21,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:21,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:21,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T22:23:21,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T22:23:21,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:21,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:21,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T22:23:21,132 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T22:23:21,133 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:21,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:21,134 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T22:23:21,136 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740 2024-12-02T22:23:21,145 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740 2024-12-02T22:23:21,161 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T22:23:21,161 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T22:23:21,166 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T22:23:21,192 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T22:23:21,196 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72460481, jitterRate=0.07974530756473541}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T22:23:21,197 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T22:23:21,198 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733178201114Writing region info on filesystem at 1733178201114Initializing all the Stores at 1733178201117 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178201117Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178201117Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178201118 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178201118Cleaning up temporary data from old regions at 1733178201161 (+43 ms)Running coprocessor post-open hooks at 1733178201197 (+36 ms)Region opened successfully at 1733178201198 (+1 ms) 2024-12-02T22:23:21,209 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733178200834 2024-12-02T22:23:21,221 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T22:23:21,221 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T22:23:21,223 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:21,225 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7ae73638d47e,38411,1733178196190, state=OPEN 2024-12-02T22:23:21,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:21,298 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:21,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:21,299 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:21,299 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:21,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:21,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:21,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:21,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:21,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T22:23:21,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7ae73638d47e,38411,1733178196190 in 641 msec 2024-12-02T22:23:21,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T22:23:21,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.8660 sec 2024-12-02T22:23:21,325 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T22:23:21,325 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T22:23:21,361 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:23:21,362 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1] 2024-12-02T22:23:21,399 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:23:21,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55327, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:23:21,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.6650 sec 2024-12-02T22:23:21,437 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733178201436, completionTime=-1 2024-12-02T22:23:21,440 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-02T22:23:21,441 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T22:23:21,524 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-02T22:23:21,525 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733178261525 2024-12-02T22:23:21,525 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733178321525 2024-12-02T22:23:21,525 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 84 msec 2024-12-02T22:23:21,527 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-02T22:23:21,550 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,42475,1733178195164-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:21,551 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,42475,1733178195164-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:21,551 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,42475,1733178195164-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:21,572 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7ae73638d47e:42475, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:21,576 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:21,582 DEBUG [master/7ae73638d47e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T22:23:21,590 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:21,639 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 5.039sec 2024-12-02T22:23:21,642 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T22:23:21,643 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T22:23:21,645 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T22:23:21,646 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T22:23:21,646 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T22:23:21,647 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,42475,1733178195164-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T22:23:21,647 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,42475,1733178195164-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T22:23:21,659 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T22:23:21,660 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T22:23:21,661 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,42475,1733178195164-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:21,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a92e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:23:21,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,42475,-1 for getting cluster id 2024-12-02T22:23:21,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:23:21,775 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9163252c-8e97-41ba-b6b3-4a5e364dbbc9' 2024-12-02T22:23:21,778 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:23:21,778 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9163252c-8e97-41ba-b6b3-4a5e364dbbc9" 2024-12-02T22:23:21,781 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480e43af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:23:21,781 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,42475,-1] 2024-12-02T22:23:21,785 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:23:21,787 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:23:21,790 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:23:21,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cee57a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:23:21,794 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:23:21,802 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1] 2024-12-02T22:23:21,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:23:21,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:23:21,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7ae73638d47e,42475,1733178195164 2024-12-02T22:23:21,849 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:38075/hbase 2024-12-02T22:23:21,875 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=358, OpenFileDescriptor=600, MaxFileDescriptor=1048576, SystemLoadAverage=659, ProcessCount=11, AvailableMemoryMB=6077 2024-12-02T22:23:21,901 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:21,904 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:21,905 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:21,912 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-10234615, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-10234615, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:21,932 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-10234615/hregion-10234615.1733178201913, exclude list is [], retry=0 2024-12-02T22:23:21,942 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:21,953 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:21,954 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:22,001 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-10234615/hregion-10234615.1733178201913 2024-12-02T22:23:22,003 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:22,003 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => eb610cc1a95bea496bdcd667e1b64d1a, NAME => 'testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:23:22,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741838_1014 (size=64) 2024-12-02T22:23:22,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741838_1014 (size=64) 2024-12-02T22:23:22,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741838_1014 (size=64) 2024-12-02T22:23:22,032 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:22,035 INFO [StoreOpener-eb610cc1a95bea496bdcd667e1b64d1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:22,038 INFO [StoreOpener-eb610cc1a95bea496bdcd667e1b64d1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb610cc1a95bea496bdcd667e1b64d1a columnFamilyName a 2024-12-02T22:23:22,038 DEBUG [StoreOpener-eb610cc1a95bea496bdcd667e1b64d1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:22,044 INFO [StoreOpener-eb610cc1a95bea496bdcd667e1b64d1a-1 {}] regionserver.HStore(327): Store=eb610cc1a95bea496bdcd667e1b64d1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:22,045 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:22,046 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:22,058 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:22,060 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:22,061 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:22,067 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:22,071 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:22,072 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eb610cc1a95bea496bdcd667e1b64d1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59420149, jitterRate=-0.11457078158855438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:23:22,074 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eb610cc1a95bea496bdcd667e1b64d1a: Writing region info on filesystem at 1733178202032Initializing all the Stores at 1733178202034 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178202034Cleaning up temporary data from old regions at 1733178202061 (+27 ms)Region opened successfully at 1733178202074 (+13 ms) 2024-12-02T22:23:22,074 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eb610cc1a95bea496bdcd667e1b64d1a, disabling compactions & flushes 2024-12-02T22:23:22,074 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a. 2024-12-02T22:23:22,074 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a. 2024-12-02T22:23:22,074 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a. after waiting 0 ms 2024-12-02T22:23:22,074 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a. 2024-12-02T22:23:22,075 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a. 2024-12-02T22:23:22,075 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eb610cc1a95bea496bdcd667e1b64d1a: Waiting for close lock at 1733178202074Disabling compacts and flushes for region at 1733178202074Disabling writes for close at 1733178202074Writing region close event to WAL at 1733178202075 (+1 ms)Closed at 1733178202075 2024-12-02T22:23:22,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741837_1013 (size=175) 2024-12-02T22:23:22,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741837_1013 (size=175) 2024-12-02T22:23:22,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741837_1013 (size=175) 2024-12-02T22:23:22,097 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:22,098 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-10234615:(num 1733178201913) 2024-12-02T22:23:22,100 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T22:23:22,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741839_1015 (size=469) 2024-12-02T22:23:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741839_1015 (size=469) 2024-12-02T22:23:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741839_1015 (size=469) 2024-12-02T22:23:22,209 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T22:23:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741840_1016 (size=322) 2024-12-02T22:23:22,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741840_1016 (size=322) 2024-12-02T22:23:22,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741840_1016 (size=322) 2024-12-02T22:23:22,336 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1, size=469 (469bytes) 2024-12-02T22:23:22,337 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-02T22:23:22,338 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-02T22:23:22,338 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1 2024-12-02T22:23:22,363 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1 after 6ms 2024-12-02T22:23:22,424 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:22,425 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1 took 93ms 2024-12-02T22:23:22,450 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1 so closing down 2024-12-02T22:23:22,450 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:22,479 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-02T22:23:22,534 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000001-wal-1.temp 2024-12-02T22:23:22,536 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:22,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741841_1017 (size=469) 2024-12-02T22:23:22,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741841_1017 (size=469) 2024-12-02T22:23:22,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741841_1017 (size=469) 2024-12-02T22:23:23,009 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-02T22:23:23,021 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002 2024-12-02T22:23:23,034 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 588 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-02T22:23:23,035 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1, journal: Splitting hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1, size=469 (469bytes) at 1733178202336Finishing writing output for hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1 so closing down at 1733178202450 (+114 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000001-wal-1.temp at 1733178202534 (+84 ms)3 split writer threads finished at 1733178202536 (+2 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733178203010 (+474 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002 at 1733178203021 (+11 ms)Processed 2 edits across 1 Regions in 588 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733178203034 (+13 ms) 2024-12-02T22:23:23,100 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2, size=322 (322bytes) 2024-12-02T22:23:23,100 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2 2024-12-02T22:23:23,101 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2 after 1ms 2024-12-02T22:23:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741831_1007 (size=1321) 2024-12-02T22:23:23,163 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:23,163 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2 took 74ms 2024-12-02T22:23:23,184 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2 so closing down 2024-12-02T22:23:23,184 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:23,191 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-02T22:23:23,239 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002-wal-2.temp 2024-12-02T22:23:23,239 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:23,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741842_1018 (size=322) 2024-12-02T22:23:23,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741842_1018 (size=322) 2024-12-02T22:23:23,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741842_1018 (size=322) 2024-12-02T22:23:23,267 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T22:23:23,319 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:23,325 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-02T22:23:23,338 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 166 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-02T22:23:23,338 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2, journal: Splitting hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2, size=322 (322bytes) at 1733178203100Finishing writing output for hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2 so closing down at 1733178203184 (+84 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002-wal-2.temp at 1733178203239 (+55 ms)3 split writer threads finished at 1733178203239Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733178203267 (+28 ms)Processed 1 edits across 1 Regions in 166 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733178203338 (+71 ms) 2024-12-02T22:23:23,338 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:23,347 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:23,373 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal.1733178203348, exclude list is [], retry=0 2024-12-02T22:23:23,382 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:23,383 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:23,383 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:23,474 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal.1733178203348 2024-12-02T22:23:23,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:23,475 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => eb610cc1a95bea496bdcd667e1b64d1a, NAME => 'testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:23,476 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:23,476 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:23,476 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:23,480 INFO [StoreOpener-eb610cc1a95bea496bdcd667e1b64d1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:23,492 INFO [StoreOpener-eb610cc1a95bea496bdcd667e1b64d1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb610cc1a95bea496bdcd667e1b64d1a columnFamilyName a 2024-12-02T22:23:23,492 DEBUG [StoreOpener-eb610cc1a95bea496bdcd667e1b64d1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:23,494 INFO [StoreOpener-eb610cc1a95bea496bdcd667e1b64d1a-1 {}] regionserver.HStore(327): Store=eb610cc1a95bea496bdcd667e1b64d1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:23,495 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:23,496 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:23,506 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:23,516 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002 2024-12-02T22:23:23,596 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:23,607 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002 2024-12-02T22:23:23,612 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eb610cc1a95bea496bdcd667e1b64d1a 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-02T22:23:23,786 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/.tmp/a/df09f1365249454aa2593cc4f93fa0ae is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733178202098/Put/seqid=0 2024-12-02T22:23:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741844_1020 (size=5170) 2024-12-02T22:23:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741844_1020 (size=5170) 2024-12-02T22:23:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741844_1020 (size=5170) 2024-12-02T22:23:24,281 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/.tmp/a/df09f1365249454aa2593cc4f93fa0ae 2024-12-02T22:23:24,403 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/.tmp/a/df09f1365249454aa2593cc4f93fa0ae as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/a/df09f1365249454aa2593cc4f93fa0ae 2024-12-02T22:23:24,449 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/a/df09f1365249454aa2593cc4f93fa0ae, entries=2, sequenceid=2, filesize=5.0 K 2024-12-02T22:23:24,470 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for eb610cc1a95bea496bdcd667e1b64d1a in 844ms, sequenceid=2, compaction requested=false; wal=null 2024-12-02T22:23:24,476 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/0000000000000000002 2024-12-02T22:23:24,477 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:24,477 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:24,481 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eb610cc1a95bea496bdcd667e1b64d1a 2024-12-02T22:23:24,490 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/eb610cc1a95bea496bdcd667e1b64d1a/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-02T22:23:24,492 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eb610cc1a95bea496bdcd667e1b64d1a; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65637777, jitterRate=-0.021920904517173767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:23:24,494 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eb610cc1a95bea496bdcd667e1b64d1a: Writing region info on filesystem at 1733178203477Initializing all the Stores at 1733178203480 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178203480Obtaining lock to block concurrent updates at 1733178203612 (+132 ms)Preparing flush snapshotting stores in eb610cc1a95bea496bdcd667e1b64d1a at 1733178203612Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733178203621 (+9 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733178201902.eb610cc1a95bea496bdcd667e1b64d1a. at 1733178203621Flushing eb610cc1a95bea496bdcd667e1b64d1a/a: creating writer at 1733178203623 (+2 ms)Flushing eb610cc1a95bea496bdcd667e1b64d1a/a: appending metadata at 1733178203747 (+124 ms)Flushing eb610cc1a95bea496bdcd667e1b64d1a/a: closing flushed file at 1733178203751 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28ef5e2a: reopening flushed file at 1733178204384 (+633 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for eb610cc1a95bea496bdcd667e1b64d1a in 844ms, sequenceid=2, compaction requested=false; wal=null at 1733178204470 (+86 ms)Cleaning up temporary data from old regions at 1733178204477 (+7 ms)Region opened successfully at 1733178204493 (+16 ms) 2024-12-02T22:23:24,544 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=372 (was 358) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:43700 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:41622 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:43682 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@6b429cf5[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:54946 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:41546 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:38075/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:54926 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=695 (was 600) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=694 (was 659) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5815 (was 6077) 2024-12-02T22:23:24,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=372, OpenFileDescriptor=697, MaxFileDescriptor=1048576, SystemLoadAverage=694, ProcessCount=11, AvailableMemoryMB=5815 2024-12-02T22:23:24,603 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:24,611 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:24,618 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:24,654 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58664288, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-58664288, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:24,686 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58664288/hregion-58664288.1733178204656, exclude list is [], retry=0 2024-12-02T22:23:24,705 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:24,713 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:24,713 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:24,783 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58664288/hregion-58664288.1733178204656 2024-12-02T22:23:24,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:24,785 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fca3ea7b2fca967b69a30f22ce7cb260, NAME => 'testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:23:24,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741846_1022 (size=64) 2024-12-02T22:23:24,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741846_1022 (size=64) 2024-12-02T22:23:24,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741846_1022 (size=64) 2024-12-02T22:23:24,865 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:24,878 INFO [StoreOpener-fca3ea7b2fca967b69a30f22ce7cb260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:24,890 INFO [StoreOpener-fca3ea7b2fca967b69a30f22ce7cb260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fca3ea7b2fca967b69a30f22ce7cb260 columnFamilyName a 2024-12-02T22:23:24,891 DEBUG [StoreOpener-fca3ea7b2fca967b69a30f22ce7cb260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:24,893 INFO [StoreOpener-fca3ea7b2fca967b69a30f22ce7cb260-1 {}] regionserver.HStore(327): Store=fca3ea7b2fca967b69a30f22ce7cb260/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:24,893 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:24,897 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:24,904 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:24,907 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:24,907 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:24,911 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:24,915 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:24,916 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fca3ea7b2fca967b69a30f22ce7cb260; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63661781, jitterRate=-0.05136553943157196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:23:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fca3ea7b2fca967b69a30f22ce7cb260: Writing region info on filesystem at 1733178204865Initializing all the Stores at 1733178204877 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178204878 (+1 ms)Cleaning up temporary data from old regions at 1733178204907 (+29 ms)Region opened successfully at 1733178204916 (+9 ms) 2024-12-02T22:23:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fca3ea7b2fca967b69a30f22ce7cb260, disabling compactions & flushes 2024-12-02T22:23:24,916 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260. 2024-12-02T22:23:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260. 2024-12-02T22:23:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260. after waiting 0 ms 2024-12-02T22:23:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260. 2024-12-02T22:23:24,921 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260. 2024-12-02T22:23:24,921 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fca3ea7b2fca967b69a30f22ce7cb260: Waiting for close lock at 1733178204916Disabling compacts and flushes for region at 1733178204916Disabling writes for close at 1733178204916Writing region close event to WAL at 1733178204921 (+5 ms)Closed at 1733178204921 2024-12-02T22:23:24,935 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-58664288/hregion-58664288.1733178204656 not finished, retry = 0 2024-12-02T22:23:24,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741845_1021 (size=175) 2024-12-02T22:23:24,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741845_1021 (size=175) 2024-12-02T22:23:24,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741845_1021 (size=175) 2024-12-02T22:23:25,052 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:25,053 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58664288:(num 1733178204656) 2024-12-02T22:23:25,062 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T22:23:25,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741847_1023 (size=469) 2024-12-02T22:23:25,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741847_1023 (size=469) 2024-12-02T22:23:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741847_1023 (size=469) 2024-12-02T22:23:25,149 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T22:23:25,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741848_1024 (size=322) 2024-12-02T22:23:25,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741848_1024 (size=322) 2024-12-02T22:23:25,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741848_1024 (size=322) 2024-12-02T22:23:25,355 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2, size=322 (322bytes) 2024-12-02T22:23:25,355 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2 2024-12-02T22:23:25,356 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2 after 1ms 2024-12-02T22:23:25,416 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:25,417 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2 took 63ms 2024-12-02T22:23:25,444 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2 so closing down 2024-12-02T22:23:25,444 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:25,450 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-02T22:23:25,530 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002-wal-2.temp 2024-12-02T22:23:25,531 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:25,551 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T22:23:25,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741849_1025 (size=322) 2024-12-02T22:23:25,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741849_1025 (size=322) 2024-12-02T22:23:25,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741849_1025 (size=322) 2024-12-02T22:23:25,619 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T22:23:25,631 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002 2024-12-02T22:23:25,631 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 206 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-02T22:23:25,631 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2, journal: Splitting hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2, size=322 (322bytes) at 1733178205355Finishing writing output for hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2 so closing down at 1733178205444 (+89 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002-wal-2.temp at 1733178205531 (+87 ms)3 split writer threads finished at 1733178205531Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733178205620 (+89 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002 at 1733178205631 (+11 ms)Processed 1 edits across 1 Regions in 206 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733178205631 2024-12-02T22:23:25,666 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1, size=469 (469bytes) 2024-12-02T22:23:25,666 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1 2024-12-02T22:23:25,669 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1 after 1ms 2024-12-02T22:23:25,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T22:23:25,689 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T22:23:25,699 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T22:23:25,699 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T22:23:25,699 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T22:23:25,699 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T22:23:25,701 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-02T22:23:25,701 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-02T22:23:25,729 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:25,729 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1 took 64ms 2024-12-02T22:23:25,734 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1 so closing down 2024-12-02T22:23:25,734 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:25,749 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-02T22:23:25,808 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000001-wal-1.temp 2024-12-02T22:23:25,809 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:25,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741850_1026 (size=469) 2024-12-02T22:23:25,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741850_1026 (size=469) 2024-12-02T22:23:25,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741850_1026 (size=469) 2024-12-02T22:23:26,271 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 1 ms) 2024-12-02T22:23:26,352 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:26,386 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002, length=322 2024-12-02T22:23:26,396 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002 2024-12-02T22:23:26,397 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 666 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-02T22:23:26,397 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1, journal: Splitting hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1, size=469 (469bytes) at 1733178205666Finishing writing output for hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1 so closing down at 1733178205734 (+68 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000001-wal-1.temp at 1733178205808 (+74 ms)3 split writer threads finished at 1733178205809 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 1 ms) at 1733178206271 (+462 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002 at 1733178206396 (+125 ms)Processed 2 edits across 1 Regions in 666 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733178206397 (+1 ms) 2024-12-02T22:23:26,397 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:26,410 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:26,439 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal.1733178206411, exclude list is [], retry=0 2024-12-02T22:23:26,457 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:26,457 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:26,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:26,547 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal.1733178206411 2024-12-02T22:23:26,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:26,549 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fca3ea7b2fca967b69a30f22ce7cb260, NAME => 'testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:26,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:26,549 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,549 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,569 INFO [StoreOpener-fca3ea7b2fca967b69a30f22ce7cb260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,577 INFO [StoreOpener-fca3ea7b2fca967b69a30f22ce7cb260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fca3ea7b2fca967b69a30f22ce7cb260 columnFamilyName a 2024-12-02T22:23:26,577 DEBUG [StoreOpener-fca3ea7b2fca967b69a30f22ce7cb260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:26,584 INFO [StoreOpener-fca3ea7b2fca967b69a30f22ce7cb260-1 {}] regionserver.HStore(327): Store=fca3ea7b2fca967b69a30f22ce7cb260/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:26,588 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,591 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,605 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,609 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002 2024-12-02T22:23:26,677 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:26,698 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:23:26,703 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002 2024-12-02T22:23:26,703 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fca3ea7b2fca967b69a30f22ce7cb260 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-02T22:23:26,778 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/.tmp/a/75827c4096e049c68b30e04e5e8da04b is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733178205053/Put/seqid=0 2024-12-02T22:23:26,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741852_1028 (size=5170) 2024-12-02T22:23:26,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741852_1028 (size=5170) 2024-12-02T22:23:26,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741852_1028 (size=5170) 2024-12-02T22:23:26,897 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/.tmp/a/75827c4096e049c68b30e04e5e8da04b 2024-12-02T22:23:26,929 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/.tmp/a/75827c4096e049c68b30e04e5e8da04b as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/a/75827c4096e049c68b30e04e5e8da04b 2024-12-02T22:23:26,952 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/a/75827c4096e049c68b30e04e5e8da04b, entries=2, sequenceid=2, filesize=5.0 K 2024-12-02T22:23:26,952 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for fca3ea7b2fca967b69a30f22ce7cb260 in 249ms, sequenceid=2, compaction requested=false; wal=null 2024-12-02T22:23:26,956 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/0000000000000000002 2024-12-02T22:23:26,956 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,956 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,960 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fca3ea7b2fca967b69a30f22ce7cb260 2024-12-02T22:23:26,969 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/fca3ea7b2fca967b69a30f22ce7cb260/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-02T22:23:26,973 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fca3ea7b2fca967b69a30f22ce7cb260; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71158204, jitterRate=0.06033986806869507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:23:26,973 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fca3ea7b2fca967b69a30f22ce7cb260: Writing region info on filesystem at 1733178206549Initializing all the Stores at 1733178206564 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178206564Obtaining lock to block concurrent updates at 1733178206703 (+139 ms)Preparing flush snapshotting stores in fca3ea7b2fca967b69a30f22ce7cb260 at 1733178206703Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733178206703Flushing stores of testReplayEditsWrittenIntoWAL,,1733178204603.fca3ea7b2fca967b69a30f22ce7cb260. at 1733178206703Flushing fca3ea7b2fca967b69a30f22ce7cb260/a: creating writer at 1733178206703Flushing fca3ea7b2fca967b69a30f22ce7cb260/a: appending metadata at 1733178206776 (+73 ms)Flushing fca3ea7b2fca967b69a30f22ce7cb260/a: closing flushed file at 1733178206776Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1137b54a: reopening flushed file at 1733178206923 (+147 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for fca3ea7b2fca967b69a30f22ce7cb260 in 249ms, sequenceid=2, compaction requested=false; wal=null at 1733178206952 (+29 ms)Cleaning up temporary data from old regions at 1733178206957 (+5 ms)Region opened successfully at 1733178206973 (+16 ms) 2024-12-02T22:23:27,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=384 (was 372) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:43700 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:41698 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:55018 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:41692 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:43782 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:54926 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=779 (was 697) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=694 (was 694), ProcessCount=11 (was 11), AvailableMemoryMB=5599 (was 5815) 2024-12-02T22:23:27,050 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=779, MaxFileDescriptor=1048576, SystemLoadAverage=694, ProcessCount=11, AvailableMemoryMB=5598 2024-12-02T22:23:27,108 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:27,119 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:27,120 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:27,136 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-46395095, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-46395095, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:27,167 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-46395095/hregion-46395095.1733178207137, exclude list is [], retry=0 2024-12-02T22:23:27,183 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:27,185 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:27,205 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:27,281 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-46395095/hregion-46395095.1733178207137 2024-12-02T22:23:27,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:23:27,285 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1fe7985944d981039fcd6290e8710b51, NAME => 'testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:23:27,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741854_1030 (size=64) 2024-12-02T22:23:27,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741854_1030 (size=64) 2024-12-02T22:23:27,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741854_1030 (size=64) 2024-12-02T22:23:27,407 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:27,421 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,425 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fe7985944d981039fcd6290e8710b51 columnFamilyName a 2024-12-02T22:23:27,425 DEBUG [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:27,428 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(327): Store=1fe7985944d981039fcd6290e8710b51/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:27,429 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,437 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fe7985944d981039fcd6290e8710b51 columnFamilyName b 2024-12-02T22:23:27,437 DEBUG [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:27,441 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(327): Store=1fe7985944d981039fcd6290e8710b51/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:27,441 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,456 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fe7985944d981039fcd6290e8710b51 columnFamilyName c 2024-12-02T22:23:27,456 DEBUG [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:27,460 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(327): Store=1fe7985944d981039fcd6290e8710b51/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:27,461 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,468 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,475 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,475 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,476 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:27,478 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:27,493 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:27,494 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1fe7985944d981039fcd6290e8710b51; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60536531, jitterRate=-0.09793539345264435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:27,494 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1fe7985944d981039fcd6290e8710b51: Writing region info on filesystem at 1733178207407Initializing all the Stores at 1733178207414 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178207414Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178207420 (+6 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178207420Cleaning up temporary data from old regions at 1733178207475 (+55 ms)Region opened successfully at 1733178207494 (+19 ms) 2024-12-02T22:23:27,494 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1fe7985944d981039fcd6290e8710b51, disabling compactions & flushes 2024-12-02T22:23:27,494 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:27,494 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:27,494 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. after waiting 0 ms 2024-12-02T22:23:27,494 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:27,497 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:27,497 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1fe7985944d981039fcd6290e8710b51: Waiting for close lock at 1733178207494Disabling compacts and flushes for region at 1733178207494Disabling writes for close at 1733178207494Writing region close event to WAL at 1733178207497 (+3 ms)Closed at 1733178207497 2024-12-02T22:23:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741853_1029 (size=175) 2024-12-02T22:23:27,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741853_1029 (size=175) 2024-12-02T22:23:27,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741853_1029 (size=175) 2024-12-02T22:23:27,557 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:27,557 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-46395095:(num 1733178207137) 2024-12-02T22:23:27,557 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:27,573 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:27,607 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574, exclude list is [], retry=0 2024-12-02T22:23:27,621 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:27,622 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:27,622 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:27,677 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 2024-12-02T22:23:27,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:23:28,414 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574, size=0 (0bytes) 2024-12-02T22:23:28,414 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 might be still open, length is 0 2024-12-02T22:23:28,414 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 2024-12-02T22:23:28,415 WARN [IPC Server handler 4 on default port 38075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-02T22:23:28,423 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 after 8ms 2024-12-02T22:23:29,231 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:60710 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34817:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60710 dst: /127.0.0.1:34817 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34817 remote=/127.0.0.1:60710]. Total timeout mills is 60000, 59089 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:29,231 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:51276 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40725:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51276 dst: /127.0.0.1:40725 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:29,233 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:35226 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37315:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35226 dst: /127.0.0.1:37315 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:29,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741855_1032 (size=497775) 2024-12-02T22:23:32,424 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 after 4010ms 2024-12-02T22:23:32,495 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:32,501 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 took 4088ms 2024-12-02T22:23:32,533 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733178207574.temp 2024-12-02T22:23:32,643 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000000001-wal.1733178207574.temp 2024-12-02T22:23:32,872 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574; continuing. 2024-12-02T22:23:32,872 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 so closing down 2024-12-02T22:23:32,872 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:32,872 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:32,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741856_1033 (size=497783) 2024-12-02T22:23:32,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741856_1033 (size=497783) 2024-12-02T22:23:32,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741856_1033 (size=497783) 2024-12-02T22:23:33,288 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000000001-wal.1733178207574.temp (wrote 3002 edits, skipped 0 edits in 55 ms) 2024-12-02T22:23:33,301 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000000001-wal.1733178207574.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000003002 2024-12-02T22:23:33,304 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 799 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T22:23:33,304 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574, journal: Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574, size=0 (0bytes) at 1733178208414Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000000001-wal.1733178207574.temp at 1733178212643 (+4229 ms)Split 1024 edits, skipped 0 edits. at 1733178212698 (+55 ms)Split 2048 edits, skipped 0 edits. at 1733178212841 (+143 ms)Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 so closing down at 1733178212872 (+31 ms)3 split writer threads finished at 1733178212873 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000000001-wal.1733178207574.temp (wrote 3002 edits, skipped 0 edits in 55 ms) at 1733178213288 (+415 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000000001-wal.1733178207574.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000003002 at 1733178213301 (+13 ms)Processed 3002 edits across 1 Regions in 799 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574, size=0, length=0, corrupted=false, cancelled=false at 1733178213304 (+3 ms) 2024-12-02T22:23:33,312 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178207574 2024-12-02T22:23:33,325 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000003002 2024-12-02T22:23:33,325 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:33,331 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:33,390 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178213331, exclude list is [], retry=0 2024-12-02T22:23:33,405 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:33,406 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:33,406 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:33,500 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178213331 2024-12-02T22:23:33,515 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:33,516 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:33,525 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:33,528 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fe7985944d981039fcd6290e8710b51 columnFamilyName a 2024-12-02T22:23:33,528 DEBUG [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:33,532 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(327): Store=1fe7985944d981039fcd6290e8710b51/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:33,533 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:33,535 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fe7985944d981039fcd6290e8710b51 columnFamilyName b 2024-12-02T22:23:33,535 DEBUG [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:33,536 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(327): Store=1fe7985944d981039fcd6290e8710b51/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:33,536 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:33,556 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fe7985944d981039fcd6290e8710b51 columnFamilyName c 2024-12-02T22:23:33,556 DEBUG [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:33,561 INFO [StoreOpener-1fe7985944d981039fcd6290e8710b51-1 {}] regionserver.HStore(327): Store=1fe7985944d981039fcd6290e8710b51/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:33,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:33,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:33,567 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:33,568 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000003002 2024-12-02T22:23:33,627 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:33,723 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T22:23:34,593 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1fe7985944d981039fcd6290e8710b51 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T22:23:34,728 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/a/5b53710e3f46437d98041a47b4beb28f is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733178207714/Put/seqid=0 2024-12-02T22:23:34,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741858_1035 (size=50463) 2024-12-02T22:23:34,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741858_1035 (size=50463) 2024-12-02T22:23:34,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741858_1035 (size=50463) 2024-12-02T22:23:34,831 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/a/5b53710e3f46437d98041a47b4beb28f 2024-12-02T22:23:34,856 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/a/5b53710e3f46437d98041a47b4beb28f as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/a/5b53710e3f46437d98041a47b4beb28f 2024-12-02T22:23:34,903 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/a/5b53710e3f46437d98041a47b4beb28f, entries=754, sequenceid=754, filesize=49.3 K 2024-12-02T22:23:34,904 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 1fe7985944d981039fcd6290e8710b51 in 312ms, sequenceid=754, compaction requested=false; wal=null 2024-12-02T22:23:34,987 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T22:23:34,988 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1fe7985944d981039fcd6290e8710b51 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T22:23:35,032 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/a/74f791062b6f4e7aa3e255c3b90d943a is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733178207851/Put/seqid=0 2024-12-02T22:23:35,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741859_1036 (size=20072) 2024-12-02T22:23:35,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741859_1036 (size=20072) 2024-12-02T22:23:35,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741859_1036 (size=20072) 2024-12-02T22:23:35,561 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/a/74f791062b6f4e7aa3e255c3b90d943a 2024-12-02T22:23:35,642 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/b/8b71d9144c624d61834cf33e27b081fd is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733178207907/Put/seqid=0 2024-12-02T22:23:35,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741860_1037 (size=35835) 2024-12-02T22:23:35,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741860_1037 (size=35835) 2024-12-02T22:23:35,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741860_1037 (size=35835) 2024-12-02T22:23:36,089 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/b/8b71d9144c624d61834cf33e27b081fd 2024-12-02T22:23:36,145 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/a/74f791062b6f4e7aa3e255c3b90d943a as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/a/74f791062b6f4e7aa3e255c3b90d943a 2024-12-02T22:23:36,206 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/a/74f791062b6f4e7aa3e255c3b90d943a, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-02T22:23:36,213 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/b/8b71d9144c624d61834cf33e27b081fd as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/b/8b71d9144c624d61834cf33e27b081fd 2024-12-02T22:23:36,231 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/b/8b71d9144c624d61834cf33e27b081fd, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-02T22:23:36,232 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1fe7985944d981039fcd6290e8710b51 in 1243ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-02T22:23:36,268 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T22:23:36,269 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1fe7985944d981039fcd6290e8710b51 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T22:23:36,309 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/b/49597e0541514b9cb8be7620967d9398 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733178208000/Put/seqid=0 2024-12-02T22:23:36,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741861_1038 (size=35082) 2024-12-02T22:23:36,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741861_1038 (size=35082) 2024-12-02T22:23:36,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741861_1038 (size=35082) 2024-12-02T22:23:36,431 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/b/49597e0541514b9cb8be7620967d9398 2024-12-02T22:23:36,550 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/c/15d2844ab8204a66ac132c8a78e9bdd8 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733178208120/Put/seqid=0 2024-12-02T22:23:36,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741862_1039 (size=20825) 2024-12-02T22:23:36,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741862_1039 (size=20825) 2024-12-02T22:23:36,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741862_1039 (size=20825) 2024-12-02T22:23:36,641 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/c/15d2844ab8204a66ac132c8a78e9bdd8 2024-12-02T22:23:36,705 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/b/49597e0541514b9cb8be7620967d9398 as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/b/49597e0541514b9cb8be7620967d9398 2024-12-02T22:23:36,715 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/b/49597e0541514b9cb8be7620967d9398, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-02T22:23:36,721 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/c/15d2844ab8204a66ac132c8a78e9bdd8 as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/c/15d2844ab8204a66ac132c8a78e9bdd8 2024-12-02T22:23:36,794 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/c/15d2844ab8204a66ac132c8a78e9bdd8, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-02T22:23:36,795 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1fe7985944d981039fcd6290e8710b51 in 526ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-02T22:23:36,824 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733178208316/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:36,843 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000003002 2024-12-02T22:23:36,856 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T22:23:36,856 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1fe7985944d981039fcd6290e8710b51 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-02T22:23:36,903 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/c/7927f1542e5e40e79576db4d2f42942e is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733178208165/Put/seqid=0 2024-12-02T22:23:36,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741863_1040 (size=50301) 2024-12-02T22:23:36,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741863_1040 (size=50301) 2024-12-02T22:23:36,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741863_1040 (size=50301) 2024-12-02T22:23:36,969 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/c/7927f1542e5e40e79576db4d2f42942e 2024-12-02T22:23:36,991 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7927f1542e5e40e79576db4d2f42942e 2024-12-02T22:23:36,993 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/.tmp/c/7927f1542e5e40e79576db4d2f42942e as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/c/7927f1542e5e40e79576db4d2f42942e 2024-12-02T22:23:37,008 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7927f1542e5e40e79576db4d2f42942e 2024-12-02T22:23:37,008 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/c/7927f1542e5e40e79576db4d2f42942e, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-02T22:23:37,008 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 1fe7985944d981039fcd6290e8710b51 in 152ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-02T22:23:37,010 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/0000000000000003002 2024-12-02T22:23:37,018 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:37,018 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:37,020 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:23:37,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1fe7985944d981039fcd6290e8710b51 2024-12-02T22:23:37,032 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenIntoWAL/1fe7985944d981039fcd6290e8710b51/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-02T22:23:37,040 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1fe7985944d981039fcd6290e8710b51; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62797737, jitterRate=-0.06424079835414886}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:23:37,040 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1fe7985944d981039fcd6290e8710b51: Writing region info on filesystem at 1733178213516Initializing all the Stores at 1733178213518 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178213518Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178213520 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178213520Cleaning up temporary data from old regions at 1733178217018 (+3498 ms)Region opened successfully at 1733178217040 (+22 ms) 2024-12-02T22:23:37,168 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1fe7985944d981039fcd6290e8710b51, disabling compactions & flushes 2024-12-02T22:23:37,168 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:37,168 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:37,168 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. after waiting 0 ms 2024-12-02T22:23:37,168 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:37,241 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733178207113.1fe7985944d981039fcd6290e8710b51. 2024-12-02T22:23:37,241 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1fe7985944d981039fcd6290e8710b51: Waiting for close lock at 1733178217168Disabling compacts and flushes for region at 1733178217168Disabling writes for close at 1733178217168Writing region close event to WAL at 1733178217240 (+72 ms)Closed at 1733178217240 2024-12-02T22:23:37,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741857_1034 (size=175) 2024-12-02T22:23:37,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741857_1034 (size=175) 2024-12-02T22:23:37,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741857_1034 (size=175) 2024-12-02T22:23:37,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:37,305 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733178213331) 2024-12-02T22:23:37,369 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=394 (was 384) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:38075 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:38075 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_630703732_22 at /127.0.0.1:60762 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_630703732_22 at /127.0.0.1:35274 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:43187 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_630703732_22 at /127.0.0.1:51294 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:38099 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43187 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_630703732_22 at /127.0.0.1:35246 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38099 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=853 (was 779) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=826 (was 694) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4776 (was 5598) 2024-12-02T22:23:37,390 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=394, OpenFileDescriptor=853, MaxFileDescriptor=1048576, SystemLoadAverage=826, ProcessCount=11, AvailableMemoryMB=4774 2024-12-02T22:23:37,476 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:37,483 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:37,484 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:37,492 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44148958, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-44148958, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:37,549 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44148958/hregion-44148958.1733178217493, exclude list is [], retry=0 2024-12-02T22:23:37,573 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:37,573 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:37,577 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:37,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44148958/hregion-44148958.1733178217493 2024-12-02T22:23:37,655 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:37,655 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 528bb068b77a263fbb210c77b2804a83, NAME => 'test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:23:37,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741865_1042 (size=43) 2024-12-02T22:23:37,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741865_1042 (size=43) 2024-12-02T22:23:37,702 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:37,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741865_1042 (size=43) 2024-12-02T22:23:37,713 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,721 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 528bb068b77a263fbb210c77b2804a83 columnFamilyName a 2024-12-02T22:23:37,721 DEBUG [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:37,728 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(327): Store=528bb068b77a263fbb210c77b2804a83/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:37,729 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,741 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 528bb068b77a263fbb210c77b2804a83 columnFamilyName b 2024-12-02T22:23:37,741 DEBUG [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:37,760 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(327): Store=528bb068b77a263fbb210c77b2804a83/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:37,760 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,767 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 528bb068b77a263fbb210c77b2804a83 columnFamilyName c 2024-12-02T22:23:37,767 DEBUG [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:37,768 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(327): Store=528bb068b77a263fbb210c77b2804a83/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:37,768 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,773 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,774 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,776 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,776 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,777 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:37,778 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:37,789 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:37,789 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 528bb068b77a263fbb210c77b2804a83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66683389, jitterRate=-0.0063400715589523315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:37,791 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 528bb068b77a263fbb210c77b2804a83: Writing region info on filesystem at 1733178217703Initializing all the Stores at 1733178217704 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178217704Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178217708 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178217708Cleaning up temporary data from old regions at 1733178217776 (+68 ms)Region opened successfully at 1733178217791 (+15 ms) 2024-12-02T22:23:37,792 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 528bb068b77a263fbb210c77b2804a83, disabling compactions & flushes 2024-12-02T22:23:37,792 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. 2024-12-02T22:23:37,792 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. 2024-12-02T22:23:37,792 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. after waiting 0 ms 2024-12-02T22:23:37,792 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. 2024-12-02T22:23:37,801 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. 2024-12-02T22:23:37,801 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 528bb068b77a263fbb210c77b2804a83: Waiting for close lock at 1733178217791Disabling compacts and flushes for region at 1733178217791Disabling writes for close at 1733178217792 (+1 ms)Writing region close event to WAL at 1733178217801 (+9 ms)Closed at 1733178217801 2024-12-02T22:23:37,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741864_1041 (size=175) 2024-12-02T22:23:37,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741864_1041 (size=175) 2024-12-02T22:23:37,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741864_1041 (size=175) 2024-12-02T22:23:37,843 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:37,843 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44148958:(num 1733178217493) 2024-12-02T22:23:37,843 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:37,846 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:37,881 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847, exclude list is [], retry=0 2024-12-02T22:23:37,886 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:37,887 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:37,887 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:37,950 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847 2024-12-02T22:23:37,950 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:23:38,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741866_1043 (size=371388) 2024-12-02T22:23:38,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741866_1043 (size=371388) 2024-12-02T22:23:38,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741866_1043 (size=371388) 2024-12-02T22:23:38,527 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847, size=362.7 K (371388bytes) 2024-12-02T22:23:38,527 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847 2024-12-02T22:23:38,530 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847 after 2ms 2024-12-02T22:23:38,609 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:38,610 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847 took 83ms 2024-12-02T22:23:38,629 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733178217847.temp 2024-12-02T22:23:38,668 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847 so closing down 2024-12-02T22:23:38,669 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:38,699 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000000001-wal.1733178217847.temp 2024-12-02T22:23:38,729 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:38,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741867_1044 (size=371388) 2024-12-02T22:23:38,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741867_1044 (size=371388) 2024-12-02T22:23:38,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741867_1044 (size=371388) 2024-12-02T22:23:38,785 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000000001-wal.1733178217847.temp (wrote 3000 edits, skipped 0 edits in 29 ms) 2024-12-02T22:23:38,794 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000000001-wal.1733178217847.temp to hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003000 2024-12-02T22:23:38,794 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 181 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-02T22:23:38,794 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847, journal: Splitting hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847, size=362.7 K (371388bytes) at 1733178218527Split 1024 edits, skipped 0 edits. at 1733178218626 (+99 ms)Split 2048 edits, skipped 0 edits. at 1733178218645 (+19 ms)Finishing writing output for hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847 so closing down at 1733178218668 (+23 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000000001-wal.1733178217847.temp at 1733178218699 (+31 ms)3 split writer threads finished at 1733178218729 (+30 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000000001-wal.1733178217847.temp (wrote 3000 edits, skipped 0 edits in 29 ms) at 1733178218785 (+56 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000000001-wal.1733178217847.temp to hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003000 at 1733178218794 (+9 ms)Processed 3000 edits across 1 Regions in 181 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733178218794 2024-12-02T22:23:38,817 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178217847 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178217847 2024-12-02T22:23:38,825 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003000 2024-12-02T22:23:38,825 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:38,828 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:38,851 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828, exclude list is [], retry=0 2024-12-02T22:23:38,874 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:38,875 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:38,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:38,931 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828 2024-12-02T22:23:38,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:23:39,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741868_1045 (size=371515) 2024-12-02T22:23:39,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741868_1045 (size=371515) 2024-12-02T22:23:39,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741868_1045 (size=371515) 2024-12-02T22:23:39,245 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828, size=362.8 K (371515bytes) 2024-12-02T22:23:39,245 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828 2024-12-02T22:23:39,246 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828 after 1ms 2024-12-02T22:23:39,306 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:39,318 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828 took 73ms 2024-12-02T22:23:39,345 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733178218828.temp 2024-12-02T22:23:39,362 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828 so closing down 2024-12-02T22:23:39,362 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:39,391 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003001-wal.1733178218828.temp 2024-12-02T22:23:39,404 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:39,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741869_1046 (size=371515) 2024-12-02T22:23:39,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741869_1046 (size=371515) 2024-12-02T22:23:39,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741869_1046 (size=371515) 2024-12-02T22:23:39,520 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003001-wal.1733178218828.temp (wrote 3000 edits, skipped 0 edits in 13 ms) 2024-12-02T22:23:39,537 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003001-wal.1733178218828.temp to hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000006000 2024-12-02T22:23:39,544 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 223 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-02T22:23:39,545 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828, journal: Splitting hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828, size=362.8 K (371515bytes) at 1733178219245Split 1024 edits, skipped 0 edits. at 1733178219332 (+87 ms)Split 2048 edits, skipped 0 edits. at 1733178219344 (+12 ms)Finishing writing output for hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828 so closing down at 1733178219362 (+18 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003001-wal.1733178218828.temp at 1733178219391 (+29 ms)3 split writer threads finished at 1733178219404 (+13 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003001-wal.1733178218828.temp (wrote 3000 edits, skipped 0 edits in 13 ms) at 1733178219521 (+117 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003001-wal.1733178218828.temp to hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000006000 at 1733178219537 (+16 ms)Processed 3000 edits across 1 Regions in 223 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733178219545 (+8 ms) 2024-12-02T22:23:39,576 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178218828 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178218828 2024-12-02T22:23:39,601 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000006000 2024-12-02T22:23:39,601 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:39,629 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/test2727-manual,16010,1733178217473, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:39,656 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178219630, exclude list is [], retry=0 2024-12-02T22:23:39,662 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:39,663 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:39,663 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:39,737 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733178217473/wal.1733178219630 2024-12-02T22:23:39,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:23:39,741 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 528bb068b77a263fbb210c77b2804a83, NAME => 'test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:39,741 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:39,743 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:39,744 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:39,753 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:39,758 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 528bb068b77a263fbb210c77b2804a83 columnFamilyName a 2024-12-02T22:23:39,758 DEBUG [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:39,769 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(327): Store=528bb068b77a263fbb210c77b2804a83/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:39,770 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:39,777 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 528bb068b77a263fbb210c77b2804a83 columnFamilyName b 2024-12-02T22:23:39,777 DEBUG [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:39,781 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(327): Store=528bb068b77a263fbb210c77b2804a83/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:39,781 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:39,785 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 528bb068b77a263fbb210c77b2804a83 columnFamilyName c 2024-12-02T22:23:39,785 DEBUG [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:39,787 INFO [StoreOpener-528bb068b77a263fbb210c77b2804a83-1 {}] regionserver.HStore(327): Store=528bb068b77a263fbb210c77b2804a83/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:39,788 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:39,789 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:39,797 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:39,798 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003000 2024-12-02T22:23:39,851 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:39,988 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003000 2024-12-02T22:23:40,001 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000006000 2024-12-02T22:23:40,062 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:40,170 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000006000 2024-12-02T22:23:40,171 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 528bb068b77a263fbb210c77b2804a83 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-02T22:23:40,251 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/a/0d5fa46c321c488bad6964c082a9f341 is 41, key is test2727/a:100/1733178218936/Put/seqid=0 2024-12-02T22:23:40,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741871_1048 (size=84227) 2024-12-02T22:23:40,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741871_1048 (size=84227) 2024-12-02T22:23:40,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741871_1048 (size=84227) 2024-12-02T22:23:40,331 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/a/0d5fa46c321c488bad6964c082a9f341 2024-12-02T22:23:40,473 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/b/c4e7d03d4ee9453bb437aef788aa13b8 is 41, key is test2727/b:100/1733178218996/Put/seqid=0 2024-12-02T22:23:40,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741872_1049 (size=84609) 2024-12-02T22:23:40,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741872_1049 (size=84609) 2024-12-02T22:23:40,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741872_1049 (size=84609) 2024-12-02T22:23:40,986 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/b/c4e7d03d4ee9453bb437aef788aa13b8 2024-12-02T22:23:41,107 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/c/6d522fdeae1f4760bcab4959354efca0 is 41, key is test2727/c:100/1733178219068/Put/seqid=0 2024-12-02T22:23:41,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741873_1050 (size=84609) 2024-12-02T22:23:41,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741873_1050 (size=84609) 2024-12-02T22:23:41,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741873_1050 (size=84609) 2024-12-02T22:23:41,213 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/c/6d522fdeae1f4760bcab4959354efca0 2024-12-02T22:23:41,248 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/a/0d5fa46c321c488bad6964c082a9f341 as hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/a/0d5fa46c321c488bad6964c082a9f341 2024-12-02T22:23:41,268 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/a/0d5fa46c321c488bad6964c082a9f341, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-02T22:23:41,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/b/c4e7d03d4ee9453bb437aef788aa13b8 as hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/b/c4e7d03d4ee9453bb437aef788aa13b8 2024-12-02T22:23:41,282 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/b/c4e7d03d4ee9453bb437aef788aa13b8, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-02T22:23:41,284 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/.tmp/c/6d522fdeae1f4760bcab4959354efca0 as hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/c/6d522fdeae1f4760bcab4959354efca0 2024-12-02T22:23:41,298 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/c/6d522fdeae1f4760bcab4959354efca0, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-02T22:23:41,298 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 528bb068b77a263fbb210c77b2804a83 in 1128ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-02T22:23:41,305 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000003000 2024-12-02T22:23:41,309 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/0000000000000006000 2024-12-02T22:23:41,311 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:41,311 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:41,312 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:41,320 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 528bb068b77a263fbb210c77b2804a83 2024-12-02T22:23:41,340 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/test2727/528bb068b77a263fbb210c77b2804a83/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-02T22:23:41,343 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 528bb068b77a263fbb210c77b2804a83; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67021912, jitterRate=-0.0012956857681274414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:41,344 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 528bb068b77a263fbb210c77b2804a83: Writing region info on filesystem at 1733178219745Initializing all the Stores at 1733178219752 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178219752Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178219753 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178219753Obtaining lock to block concurrent updates at 1733178220171 (+418 ms)Preparing flush snapshotting stores in 528bb068b77a263fbb210c77b2804a83 at 1733178220171Finished memstore snapshotting test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733178220171Flushing stores of test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. at 1733178220171Flushing 528bb068b77a263fbb210c77b2804a83/a: creating writer at 1733178220171Flushing 528bb068b77a263fbb210c77b2804a83/a: appending metadata at 1733178220250 (+79 ms)Flushing 528bb068b77a263fbb210c77b2804a83/a: closing flushed file at 1733178220250Flushing 528bb068b77a263fbb210c77b2804a83/b: creating writer at 1733178220355 (+105 ms)Flushing 528bb068b77a263fbb210c77b2804a83/b: appending metadata at 1733178220459 (+104 ms)Flushing 528bb068b77a263fbb210c77b2804a83/b: closing flushed file at 1733178220459Flushing 528bb068b77a263fbb210c77b2804a83/c: creating writer at 1733178221060 (+601 ms)Flushing 528bb068b77a263fbb210c77b2804a83/c: appending metadata at 1733178221106 (+46 ms)Flushing 528bb068b77a263fbb210c77b2804a83/c: closing flushed file at 1733178221106Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@793cdd91: reopening flushed file at 1733178221237 (+131 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d0e719b: reopening flushed file at 1733178221269 (+32 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f9190bd: reopening flushed file at 1733178221283 (+14 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 528bb068b77a263fbb210c77b2804a83 in 1128ms, sequenceid=6000, compaction requested=false; wal=null at 1733178221299 (+16 ms)Cleaning up temporary data from old regions at 1733178221311 (+12 ms)Region opened successfully at 1733178221344 (+33 ms) 2024-12-02T22:23:41,346 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-02T22:23:41,346 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 528bb068b77a263fbb210c77b2804a83, disabling compactions & flushes 2024-12-02T22:23:41,346 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. 2024-12-02T22:23:41,346 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. 2024-12-02T22:23:41,346 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. after waiting 0 ms 2024-12-02T22:23:41,346 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. 2024-12-02T22:23:41,364 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733178217477.528bb068b77a263fbb210c77b2804a83. 2024-12-02T22:23:41,364 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 528bb068b77a263fbb210c77b2804a83: Waiting for close lock at 1733178221346Disabling compacts and flushes for region at 1733178221346Disabling writes for close at 1733178221346Writing region close event to WAL at 1733178221364 (+18 ms)Closed at 1733178221364 2024-12-02T22:23:41,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741870_1047 (size=175) 2024-12-02T22:23:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741870_1047 (size=175) 2024-12-02T22:23:41,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741870_1047 (size=175) 2024-12-02T22:23:41,415 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:41,415 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733178219630) 2024-12-02T22:23:41,452 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=394 (was 394), OpenFileDescriptor=911 (was 853) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=848 (was 826) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4487 (was 4774) 2024-12-02T22:23:41,480 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=394, OpenFileDescriptor=911, MaxFileDescriptor=1048576, SystemLoadAverage=848, ProcessCount=11, AvailableMemoryMB=4486 2024-12-02T22:23:41,541 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:41,609 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:41,616 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733178221609 2024-12-02T22:23:41,777 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609 2024-12-02T22:23:41,799 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:23:41,804 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c8a13aae43e5c2c7ca923e5e7902ad5c, NAME => 'testSequentialEditLogSeqNum,,1733178221546.c8a13aae43e5c2c7ca923e5e7902ad5c.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:41,805 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733178221546.c8a13aae43e5c2c7ca923e5e7902ad5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:41,805 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,805 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,808 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c doesn't exist for region: c8a13aae43e5c2c7ca923e5e7902ad5c on table testSequentialEditLogSeqNum 2024-12-02T22:23:41,811 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: c8a13aae43e5c2c7ca923e5e7902ad5c on table testSequentialEditLogSeqNum 2024-12-02T22:23:41,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741875_1052 (size=62) 2024-12-02T22:23:41,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741875_1052 (size=62) 2024-12-02T22:23:41,912 INFO [StoreOpener-c8a13aae43e5c2c7ca923e5e7902ad5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741875_1052 (size=62) 2024-12-02T22:23:41,918 INFO [StoreOpener-c8a13aae43e5c2c7ca923e5e7902ad5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8a13aae43e5c2c7ca923e5e7902ad5c columnFamilyName a 2024-12-02T22:23:41,919 DEBUG [StoreOpener-c8a13aae43e5c2c7ca923e5e7902ad5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:41,919 INFO [StoreOpener-c8a13aae43e5c2c7ca923e5e7902ad5c-1 {}] regionserver.HStore(327): Store=c8a13aae43e5c2c7ca923e5e7902ad5c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:41,919 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,920 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,921 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,923 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,923 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,936 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c8a13aae43e5c2c7ca923e5e7902ad5c 2024-12-02T22:23:41,948 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:41,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c8a13aae43e5c2c7ca923e5e7902ad5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68574831, jitterRate=0.021844610571861267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:23:41,954 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c8a13aae43e5c2c7ca923e5e7902ad5c: Writing region info on filesystem at 1733178221805Initializing all the Stores at 1733178221912 (+107 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178221912Cleaning up temporary data from old regions at 1733178221923 (+11 ms)Region opened successfully at 1733178221954 (+31 ms) 2024-12-02T22:23:42,007 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c8a13aae43e5c2c7ca923e5e7902ad5c 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-02T22:23:42,118 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/.tmp/a/2491c9a5a16d45efb0d95fa7d139011c is 81, key is testSequentialEditLogSeqNum/a:x0/1733178221954/Put/seqid=0 2024-12-02T22:23:42,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741876_1053 (size=5833) 2024-12-02T22:23:42,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741876_1053 (size=5833) 2024-12-02T22:23:42,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741876_1053 (size=5833) 2024-12-02T22:23:42,193 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/.tmp/a/2491c9a5a16d45efb0d95fa7d139011c 2024-12-02T22:23:42,245 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/.tmp/a/2491c9a5a16d45efb0d95fa7d139011c as hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/a/2491c9a5a16d45efb0d95fa7d139011c 2024-12-02T22:23:42,286 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/a/2491c9a5a16d45efb0d95fa7d139011c, entries=10, sequenceid=13, filesize=5.7 K 2024-12-02T22:23:42,301 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c8a13aae43e5c2c7ca923e5e7902ad5c in 294ms, sequenceid=13, compaction requested=false 2024-12-02T22:23:42,301 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c8a13aae43e5c2c7ca923e5e7902ad5c: 2024-12-02T22:23:42,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T22:23:42,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T22:23:42,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T22:23:42,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T22:23:42,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T22:23:42,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741874_1051 (size=3256) 2024-12-02T22:23:42,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741874_1051 (size=3256) 2024-12-02T22:23:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741874_1051 (size=3256) 2024-12-02T22:23:42,483 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609, size=3.2 K (3256bytes) 2024-12-02T22:23:42,483 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609 2024-12-02T22:23:42,492 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609 after 9ms 2024-12-02T22:23:42,610 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:42,611 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609 took 130ms 2024-12-02T22:23:42,629 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609 so closing down 2024-12-02T22:23:42,629 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:42,677 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733178221609.temp 2024-12-02T22:23:42,763 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/0000000000000000003-wal.1733178221609.temp 2024-12-02T22:23:42,773 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:42,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741877_1054 (size=2650) 2024-12-02T22:23:42,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741877_1054 (size=2650) 2024-12-02T22:23:42,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741877_1054 (size=2650) 2024-12-02T22:23:42,861 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/0000000000000000003-wal.1733178221609.temp (wrote 15 edits, skipped 0 edits in 8 ms) 2024-12-02T22:23:42,880 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/0000000000000000003-wal.1733178221609.temp to hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/0000000000000000020 2024-12-02T22:23:42,883 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 263 ms; skipped=2; WAL=hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-02T22:23:42,884 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609, journal: Splitting hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609, size=3.2 K (3256bytes) at 1733178222483Finishing writing output for hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609 so closing down at 1733178222629 (+146 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/0000000000000000003-wal.1733178221609.temp at 1733178222763 (+134 ms)3 split writer threads finished at 1733178222773 (+10 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/0000000000000000003-wal.1733178221609.temp (wrote 15 edits, skipped 0 edits in 8 ms) at 1733178222861 (+88 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/0000000000000000003-wal.1733178221609.temp to hdfs://localhost:38075/hbase/data/default/testSequentialEditLogSeqNum/c8a13aae43e5c2c7ca923e5e7902ad5c/recovered.edits/0000000000000000020 at 1733178222880 (+19 ms)Processed 17 edits across 1 Regions in 263 ms; skipped=2; WAL=hdfs://localhost:38075/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733178221533/wal.1733178221609, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733178222883 (+3 ms) 2024-12-02T22:23:42,936 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=399 (was 394) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:34662 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:46840 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:48736 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=947 (was 911) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 848) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4294 (was 4486) 2024-12-02T22:23:42,987 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=399, OpenFileDescriptor=947, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4293 2024-12-02T22:23:43,054 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:43,070 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:43,071 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:43,093 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-09148990, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-09148990, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:43,114 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-09148990/hregion-09148990.1733178223093, exclude list is [], retry=0 2024-12-02T22:23:43,138 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:43,139 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:43,142 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:43,242 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-09148990/hregion-09148990.1733178223093 2024-12-02T22:23:43,245 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:23:43,245 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 21b09cb2be6fe175397415f5ab9fb2ae, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:23:43,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741879_1056 (size=70) 2024-12-02T22:23:43,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741879_1056 (size=70) 2024-12-02T22:23:43,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741879_1056 (size=70) 2024-12-02T22:23:43,381 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:43,410 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,434 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName a 2024-12-02T22:23:43,434 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:43,445 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:43,445 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,455 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName b 2024-12-02T22:23:43,455 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:43,456 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:43,457 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,462 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName c 2024-12-02T22:23:43,462 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:43,464 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:43,464 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,467 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,467 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,468 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:43,469 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,481 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:43,481 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 21b09cb2be6fe175397415f5ab9fb2ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59150964, jitterRate=-0.11858195066452026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:43,483 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 21b09cb2be6fe175397415f5ab9fb2ae: Writing region info on filesystem at 1733178223381Initializing all the Stores at 1733178223393 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178223393Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178223410 (+17 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178223410Cleaning up temporary data from old regions at 1733178223467 (+57 ms)Region opened successfully at 1733178223483 (+16 ms) 2024-12-02T22:23:43,483 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 21b09cb2be6fe175397415f5ab9fb2ae, disabling compactions & flushes 2024-12-02T22:23:43,483 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. 2024-12-02T22:23:43,483 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. 2024-12-02T22:23:43,483 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. after waiting 0 ms 2024-12-02T22:23:43,483 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. 2024-12-02T22:23:43,485 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. 2024-12-02T22:23:43,485 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 21b09cb2be6fe175397415f5ab9fb2ae: Waiting for close lock at 1733178223483Disabling compacts and flushes for region at 1733178223483Disabling writes for close at 1733178223483Writing region close event to WAL at 1733178223485 (+2 ms)Closed at 1733178223485 2024-12-02T22:23:43,515 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-09148990/hregion-09148990.1733178223093 not finished, retry = 0 2024-12-02T22:23:43,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741878_1055 (size=175) 2024-12-02T22:23:43,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741878_1055 (size=175) 2024-12-02T22:23:43,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741878_1055 (size=175) 2024-12-02T22:23:43,624 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:43,624 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-09148990:(num 1733178223093) 2024-12-02T22:23:43,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:43,629 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:43,667 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630, exclude list is [], retry=0 2024-12-02T22:23:43,685 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:43,686 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:43,686 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:43,753 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 2024-12-02T22:23:43,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:43,756 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 21b09cb2be6fe175397415f5ab9fb2ae, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:43,757 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:43,757 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,757 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,772 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,774 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName a 2024-12-02T22:23:43,774 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:43,775 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:43,775 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,777 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName b 2024-12-02T22:23:43,777 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:43,779 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:43,780 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,782 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName c 2024-12-02T22:23:43,782 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:43,783 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:43,784 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,785 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,787 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,788 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,788 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,791 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:43,798 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:43,800 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 21b09cb2be6fe175397415f5ab9fb2ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67178956, jitterRate=0.00104445219039917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:43,801 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 21b09cb2be6fe175397415f5ab9fb2ae: Writing region info on filesystem at 1733178223768Initializing all the Stores at 1733178223771 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178223771Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178223772 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178223772Cleaning up temporary data from old regions at 1733178223788 (+16 ms)Region opened successfully at 1733178223801 (+13 ms) 2024-12-02T22:23:43,811 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733178223810/Put/seqid=0 2024-12-02T22:23:43,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741881_1058 (size=4826) 2024-12-02T22:23:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741881_1058 (size=4826) 2024-12-02T22:23:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741881_1058 (size=4826) 2024-12-02T22:23:44,201 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T22:23:44,269 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38075/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 21b09cb2be6fe175397415f5ab9fb2ae/a 2024-12-02T22:23:44,314 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-02T22:23:44,315 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T22:23:44,315 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 21b09cb2be6fe175397415f5ab9fb2ae: 2024-12-02T22:23:44,317 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/a/4a903f340bd04573b2a5806ba81e3df6_SeqId_3_ 2024-12-02T22:23:44,318 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38075/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 21b09cb2be6fe175397415f5ab9fb2ae/a as hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/a/4a903f340bd04573b2a5806ba81e3df6_SeqId_3_ - updating store file list. 2024-12-02T22:23:44,327 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4a903f340bd04573b2a5806ba81e3df6_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-02T22:23:44,327 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/a/4a903f340bd04573b2a5806ba81e3df6_SeqId_3_ into 21b09cb2be6fe175397415f5ab9fb2ae/a 2024-12-02T22:23:44,327 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38075/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 21b09cb2be6fe175397415f5ab9fb2ae/a (new location: hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/a/4a903f340bd04573b2a5806ba81e3df6_SeqId_3_) 2024-12-02T22:23:44,412 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630, size=0 (0bytes) 2024-12-02T22:23:44,413 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 might be still open, length is 0 2024-12-02T22:23:44,413 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 2024-12-02T22:23:44,413 WARN [IPC Server handler 3 on default port 38075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-02T22:23:44,414 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 after 1ms 2024-12-02T22:23:45,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-02T22:23:45,685 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-02T22:23:45,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-02T22:23:45,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-02T22:23:47,198 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:48822 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34817:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48822 dst: /127.0.0.1:34817 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34817 remote=/127.0.0.1:48822]. Total timeout mills is 60000, 57137 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:47,200 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:46906 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37315:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46906 dst: /127.0.0.1:37315 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37315 remote=/127.0.0.1:46906]. Total timeout mills is 60000, 57136 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:47,203 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:34742 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40725:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34742 dst: /127.0.0.1:40725 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:47,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741880_1059 (size=606) 2024-12-02T22:23:47,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741880_1059 (size=606) 2024-12-02T22:23:47,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741880_1059 (size=606) 2024-12-02T22:23:48,424 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 after 4011ms 2024-12-02T22:23:48,538 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:48,538 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 took 4126ms 2024-12-02T22:23:48,583 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630; continuing. 2024-12-02T22:23:48,583 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 so closing down 2024-12-02T22:23:48,583 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:48,597 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733178223630.temp 2024-12-02T22:23:48,657 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005-wal.1733178223630.temp 2024-12-02T22:23:48,657 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741882_1060 (size=329) 2024-12-02T22:23:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741882_1060 (size=329) 2024-12-02T22:23:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741882_1060 (size=329) 2024-12-02T22:23:49,125 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005-wal.1733178223630.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T22:23:49,128 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005-wal.1733178223630.temp to hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005 2024-12-02T22:23:49,128 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 561 ms; skipped=1; WAL=hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T22:23:49,129 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630, journal: Splitting hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630, size=0 (0bytes) at 1733178224412Finishing writing output for hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 so closing down at 1733178228583 (+4171 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005-wal.1733178223630.temp at 1733178228657 (+74 ms)3 split writer threads finished at 1733178228657Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005-wal.1733178223630.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733178229125 (+468 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005-wal.1733178223630.temp to hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005 at 1733178229128 (+3 ms)Processed 2 edits across 1 Regions in 561 ms; skipped=1; WAL=hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630, size=0, length=0, corrupted=false, cancelled=false at 1733178229128 2024-12-02T22:23:49,132 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178223630 2024-12-02T22:23:49,136 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005 2024-12-02T22:23:49,137 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:49,139 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:49,161 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178229139, exclude list is [], retry=0 2024-12-02T22:23:49,177 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:49,177 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:49,178 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:49,280 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178229139 2024-12-02T22:23:49,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:23:49,282 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 21b09cb2be6fe175397415f5ab9fb2ae, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:49,282 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:49,282 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,282 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,304 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,305 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName a 2024-12-02T22:23:49,306 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:49,372 DEBUG [StoreFileOpener-21b09cb2be6fe175397415f5ab9fb2ae-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4a903f340bd04573b2a5806ba81e3df6_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-02T22:23:49,372 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/a/4a903f340bd04573b2a5806ba81e3df6_SeqId_3_ 2024-12-02T22:23:49,373 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:49,373 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,377 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName b 2024-12-02T22:23:49,377 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:49,379 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:49,379 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,380 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b09cb2be6fe175397415f5ab9fb2ae columnFamilyName c 2024-12-02T22:23:49,380 DEBUG [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:49,381 INFO [StoreOpener-21b09cb2be6fe175397415f5ab9fb2ae-1 {}] regionserver.HStore(327): Store=21b09cb2be6fe175397415f5ab9fb2ae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:49,384 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,392 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005 2024-12-02T22:23:49,476 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:49,479 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005 2024-12-02T22:23:49,479 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 21b09cb2be6fe175397415f5ab9fb2ae 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-02T22:23:49,514 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/.tmp/a/7a5dc6b7d50f49999a4f91f41d5c7e07 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733178224333/Put/seqid=0 2024-12-02T22:23:49,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741884_1062 (size=5149) 2024-12-02T22:23:49,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741884_1062 (size=5149) 2024-12-02T22:23:49,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741884_1062 (size=5149) 2024-12-02T22:23:49,549 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/.tmp/a/7a5dc6b7d50f49999a4f91f41d5c7e07 2024-12-02T22:23:49,558 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/.tmp/a/7a5dc6b7d50f49999a4f91f41d5c7e07 as hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/a/7a5dc6b7d50f49999a4f91f41d5c7e07 2024-12-02T22:23:49,567 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/a/7a5dc6b7d50f49999a4f91f41d5c7e07, entries=1, sequenceid=5, filesize=5.0 K 2024-12-02T22:23:49,567 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 21b09cb2be6fe175397415f5ab9fb2ae in 88ms, sequenceid=5, compaction requested=false; wal=null 2024-12-02T22:23:49,568 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/0000000000000000005 2024-12-02T22:23:49,570 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,570 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,571 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:49,573 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 21b09cb2be6fe175397415f5ab9fb2ae 2024-12-02T22:23:49,584 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/21b09cb2be6fe175397415f5ab9fb2ae/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-02T22:23:49,585 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 21b09cb2be6fe175397415f5ab9fb2ae; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72699726, jitterRate=0.0833103358745575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:49,586 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 21b09cb2be6fe175397415f5ab9fb2ae: Writing region info on filesystem at 1733178229282Initializing all the Stores at 1733178229285 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178229285Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178229303 (+18 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178229303Obtaining lock to block concurrent updates at 1733178229479 (+176 ms)Preparing flush snapshotting stores in 21b09cb2be6fe175397415f5ab9fb2ae at 1733178229479Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733178229480 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. at 1733178229480Flushing 21b09cb2be6fe175397415f5ab9fb2ae/a: creating writer at 1733178229480Flushing 21b09cb2be6fe175397415f5ab9fb2ae/a: appending metadata at 1733178229513 (+33 ms)Flushing 21b09cb2be6fe175397415f5ab9fb2ae/a: closing flushed file at 1733178229513Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3daa9fe1: reopening flushed file at 1733178229557 (+44 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 21b09cb2be6fe175397415f5ab9fb2ae in 88ms, sequenceid=5, compaction requested=false; wal=null at 1733178229567 (+10 ms)Cleaning up temporary data from old regions at 1733178229570 (+3 ms)Region opened successfully at 1733178229586 (+16 ms) 2024-12-02T22:23:49,591 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 21b09cb2be6fe175397415f5ab9fb2ae, disabling compactions & flushes 2024-12-02T22:23:49,591 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. 2024-12-02T22:23:49,591 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. 2024-12-02T22:23:49,591 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. after waiting 0 ms 2024-12-02T22:23:49,591 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. 2024-12-02T22:23:49,593 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733178223057.21b09cb2be6fe175397415f5ab9fb2ae. 2024-12-02T22:23:49,593 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 21b09cb2be6fe175397415f5ab9fb2ae: Waiting for close lock at 1733178229591Disabling compacts and flushes for region at 1733178229591Disabling writes for close at 1733178229591Writing region close event to WAL at 1733178229593 (+2 ms)Closed at 1733178229593 2024-12-02T22:23:49,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741883_1061 (size=175) 2024-12-02T22:23:49,613 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178229139 not finished, retry = 0 2024-12-02T22:23:49,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741883_1061 (size=175) 2024-12-02T22:23:49,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741883_1061 (size=175) 2024-12-02T22:23:49,719 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:49,719 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733178229139) 2024-12-02T22:23:49,743 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408 (was 399) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386706026_22 at /127.0.0.1:51474 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Shutdown-0 java.base@17.0.11/jdk.internal.misc.Unsafe.unpark(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:177) java.base@17.0.11/java.util.concurrent.FutureTask.finishCompletion(FutureTask.java:369) java.base@17.0.11/java.util.concurrent.FutureTask.set(FutureTask.java:232) java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:272) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:38075 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1386706026_22 at /127.0.0.1:41370 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:38075 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1013 (was 947) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=909 (was 892) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3971 (was 4293) 2024-12-02T22:23:49,767 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=406, OpenFileDescriptor=1013, MaxFileDescriptor=1048576, SystemLoadAverage=909, ProcessCount=11, AvailableMemoryMB=3969 2024-12-02T22:23:49,797 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:49,804 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T22:23:49,811 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ae73638d47e,42475,1733178195164 2024-12-02T22:23:49,815 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d93f1c6 2024-12-02T22:23:49,816 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T22:23:49,819 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T22:23:49,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:23:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-02T22:23:49,841 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T22:23:49,844 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:49,847 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T22:23:49,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-02T22:23:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:23:49,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741885_1063 (size=694) 2024-12-02T22:23:49,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741885_1063 (size=694) 2024-12-02T22:23:49,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741885_1063 (size=694) 2024-12-02T22:23:49,937 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f517d31aff1d4768327a1ec4cf716a7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3 2024-12-02T22:23:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:23:50,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741886_1064 (size=77) 2024-12-02T22:23:50,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741886_1064 (size=77) 2024-12-02T22:23:50,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741886_1064 (size=77) 2024-12-02T22:23:50,105 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:50,105 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 5f517d31aff1d4768327a1ec4cf716a7, disabling compactions & flushes 2024-12-02T22:23:50,105 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:50,105 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:50,105 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. after waiting 0 ms 2024-12-02T22:23:50,105 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:50,105 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:50,105 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f517d31aff1d4768327a1ec4cf716a7: Waiting for close lock at 1733178230105Disabling compacts and flushes for region at 1733178230105Disabling writes for close at 1733178230105Writing region close event to WAL at 1733178230105Closed at 1733178230105 2024-12-02T22:23:50,111 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T22:23:50,120 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733178230111"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733178230111"}]},"ts":"1733178230111"} 2024-12-02T22:23:50,130 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T22:23:50,140 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T22:23:50,151 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733178230140"}]},"ts":"1733178230140"} 2024-12-02T22:23:50,160 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-02T22:23:50,160 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {7ae73638d47e=0} racks are {/default-rack=0} 2024-12-02T22:23:50,164 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T22:23:50,164 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T22:23:50,164 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T22:23:50,164 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T22:23:50,164 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T22:23:50,164 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T22:23:50,164 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T22:23:50,164 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T22:23:50,164 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T22:23:50,164 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T22:23:50,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN}] 2024-12-02T22:23:50,169 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN 2024-12-02T22:23:50,171 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN; state=OFFLINE, location=7ae73638d47e,38411,1733178196190; forceNewPlan=false, retain=false 2024-12-02T22:23:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:23:50,325 INFO [7ae73638d47e:42475 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T22:23:50,326 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=OPENING, regionLocation=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:50,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN because future has completed 2024-12-02T22:23:50,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190}] 2024-12-02T22:23:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:23:50,557 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:50,557 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f517d31aff1d4768327a1ec4cf716a7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:50,558 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,558 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:50,558 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,558 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,570 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,581 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf1 2024-12-02T22:23:50,581 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:50,587 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:50,587 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,609 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf2 2024-12-02T22:23:50,609 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:50,616 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:50,620 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,622 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,623 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,629 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,629 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,633 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T22:23:50,658 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,676 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:50,678 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5f517d31aff1d4768327a1ec4cf716a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72496979, jitterRate=0.08028917014598846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T22:23:50,679 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:50,681 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5f517d31aff1d4768327a1ec4cf716a7: Running coprocessor pre-open hook at 1733178230558Writing region info on filesystem at 1733178230558Initializing all the Stores at 1733178230567 (+9 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178230567Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178230569 (+2 ms)Cleaning up temporary data from old regions at 1733178230629 (+60 ms)Running coprocessor post-open hooks at 1733178230679 (+50 ms)Region opened successfully at 1733178230681 (+2 ms) 2024-12-02T22:23:50,696 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., pid=6, masterSystemTime=1733178230516 2024-12-02T22:23:50,704 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:50,704 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:50,704 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:50,708 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7ae73638d47e,38411,1733178196190, table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T22:23:50,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190 because future has completed 2024-12-02T22:23:50,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T22:23:50,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190 in 378 msec 2024-12-02T22:23:50,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T22:23:50,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN in 579 msec 2024-12-02T22:23:50,755 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T22:23:50,756 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733178230755"}]},"ts":"1733178230755"} 2024-12-02T22:23:50,769 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-02T22:23:50,774 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T22:23:50,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 948 msec 2024-12-02T22:23:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:23:51,029 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-02T22:23:51,031 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-02T22:23:51,033 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T22:23:51,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-02T22:23:51,059 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T22:23:51,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-02T22:23:51,072 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=2] 2024-12-02T22:23:51,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=5f517d31aff1d4768327a1ec4cf716a7, source=7ae73638d47e,38411,1733178196190, destination=7ae73638d47e,35511,1733178196345, warming up region on 7ae73638d47e,35511,1733178196345 2024-12-02T22:23:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T22:23:51,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=5f517d31aff1d4768327a1ec4cf716a7, source=7ae73638d47e,38411,1733178196190, destination=7ae73638d47e,35511,1733178196345, running balancer 2024-12-02T22:23:51,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49361, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T22:23:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE 2024-12-02T22:23:51,121 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE 2024-12-02T22:23:51,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5f517d31aff1d4768327a1ec4cf716a7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:51,126 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=CLOSING, regionLocation=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:51,137 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:51,140 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf1 2024-12-02T22:23:51,140 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:51,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE because future has completed 2024-12-02T22:23:51,142 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:51,142 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:51,145 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf2 2024-12-02T22:23:51,145 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:51,150 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:23:51,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190}] 2024-12-02T22:23:51,152 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(1722): Closing 5f517d31aff1d4768327a1ec4cf716a7, disabling compactions & flushes 2024-12-02T22:23:51,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. after waiting 0 ms 2024-12-02T22:23:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] regionserver.HRegion(1676): Region close journal for 5f517d31aff1d4768327a1ec4cf716a7: Waiting for close lock at 1733178231152Disabling compacts and flushes for region at 1733178231152Disabling writes for close at 1733178231153 (+1 ms)Writing region close event to WAL at 1733178231165 (+12 ms)Closed at 1733178231165 2024-12-02T22:23:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-02T22:23:51,353 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:51,353 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:23:51,354 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 5f517d31aff1d4768327a1ec4cf716a7, disabling compactions & flushes 2024-12-02T22:23:51,354 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,354 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,354 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. after waiting 0 ms 2024-12-02T22:23:51,354 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,354 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 5f517d31aff1d4768327a1ec4cf716a7 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-02T22:23:51,413 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf1/1edaf893bf514749b6ec4d2253c197ad is 35, key is r1/cf1:q/1733178231082/Put/seqid=0 2024-12-02T22:23:51,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741887_1065 (size=4783) 2024-12-02T22:23:51,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741887_1065 (size=4783) 2024-12-02T22:23:51,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741887_1065 (size=4783) 2024-12-02T22:23:51,493 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf1/1edaf893bf514749b6ec4d2253c197ad 2024-12-02T22:23:51,508 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf1/1edaf893bf514749b6ec4d2253c197ad as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/1edaf893bf514749b6ec4d2253c197ad 2024-12-02T22:23:51,516 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/1edaf893bf514749b6ec4d2253c197ad, entries=1, sequenceid=5, filesize=4.7 K 2024-12-02T22:23:51,520 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 5f517d31aff1d4768327a1ec4cf716a7 in 166ms, sequenceid=5, compaction requested=false 2024-12-02T22:23:51,520 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-02T22:23:51,536 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-02T22:23:51,542 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,544 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 5f517d31aff1d4768327a1ec4cf716a7: Waiting for close lock at 1733178231354Running coprocessor pre-close hooks at 1733178231354Disabling compacts and flushes for region at 1733178231354Disabling writes for close at 1733178231354Obtaining lock to block concurrent updates at 1733178231354Preparing flush snapshotting stores in 5f517d31aff1d4768327a1ec4cf716a7 at 1733178231354Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733178231355 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. at 1733178231360 (+5 ms)Flushing 5f517d31aff1d4768327a1ec4cf716a7/cf1: creating writer at 1733178231360Flushing 5f517d31aff1d4768327a1ec4cf716a7/cf1: appending metadata at 1733178231413 (+53 ms)Flushing 5f517d31aff1d4768327a1ec4cf716a7/cf1: closing flushed file at 1733178231413Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e450384: reopening flushed file at 1733178231507 (+94 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 5f517d31aff1d4768327a1ec4cf716a7 in 166ms, sequenceid=5, compaction requested=false at 1733178231520 (+13 ms)Writing region close event to WAL at 1733178231522 (+2 ms)Running coprocessor post-close hooks at 1733178231536 (+14 ms)Closed at 1733178231542 (+6 ms) 2024-12-02T22:23:51,544 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 5f517d31aff1d4768327a1ec4cf716a7 move to 7ae73638d47e,35511,1733178196345 record at close sequenceid=5 2024-12-02T22:23:51,550 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:51,553 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=CLOSED 2024-12-02T22:23:51,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190 because future has completed 2024-12-02T22:23:51,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T22:23:51,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190 in 434 msec 2024-12-02T22:23:51,617 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE; state=CLOSED, location=7ae73638d47e,35511,1733178196345; forceNewPlan=false, retain=false 2024-12-02T22:23:51,769 INFO [7ae73638d47e:42475 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T22:23:51,773 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=OPENING, regionLocation=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:51,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE because future has completed 2024-12-02T22:23:51,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345}] 2024-12-02T22:23:51,989 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:51,989 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f517d31aff1d4768327a1ec4cf716a7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:51,990 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:51,990 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:51,995 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:51,995 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,011 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,019 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf1 2024-12-02T22:23:52,019 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:52,041 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/1edaf893bf514749b6ec4d2253c197ad 2024-12-02T22:23:52,041 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:52,042 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,047 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf2 2024-12-02T22:23:52,047 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:52,048 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:52,048 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,049 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,062 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,065 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,065 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,069 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T22:23:52,085 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,089 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5f517d31aff1d4768327a1ec4cf716a7; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59129712, jitterRate=-0.11889863014221191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T22:23:52,089 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:52,090 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5f517d31aff1d4768327a1ec4cf716a7: Running coprocessor pre-open hook at 1733178231996Writing region info on filesystem at 1733178231996Initializing all the Stores at 1733178232004 (+8 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178232004Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178232011 (+7 ms)Cleaning up temporary data from old regions at 1733178232065 (+54 ms)Running coprocessor post-open hooks at 1733178232090 (+25 ms)Region opened successfully at 1733178232090 2024-12-02T22:23:52,097 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., pid=9, masterSystemTime=1733178231972 2024-12-02T22:23:52,103 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:52,103 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:52,105 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=OPEN, openSeqNum=9, regionLocation=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:52,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345 because future has completed 2024-12-02T22:23:52,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-02T22:23:52,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345 in 316 msec 2024-12-02T22:23:52,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE in 1.0250 sec 2024-12-02T22:23:52,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T22:23:52,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T22:23:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:38570 deadline: 1733178292209, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7ae73638d47e port=35511 startCode=1733178196345. As of locationSeqNum=5. 2024-12-02T22:23:52,266 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7ae73638d47e port=35511 startCode=1733178196345. As of locationSeqNum=5. 2024-12-02T22:23:52,267 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7ae73638d47e port=35511 startCode=1733178196345. As of locationSeqNum=5. 2024-12-02T22:23:52,267 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,35511,1733178196345, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7ae73638d47e port=35511 startCode=1733178196345. As of locationSeqNum=5. 2024-12-02T22:23:52,381 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:23:52,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:23:52,455 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5f517d31aff1d4768327a1ec4cf716a7 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-02T22:23:52,501 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf1/06d02d3164644c0fb5a4cb60a033650a is 29, key is r1/cf1:/1733178232394/DeleteFamily/seqid=0 2024-12-02T22:23:52,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741888_1066 (size=4906) 2024-12-02T22:23:52,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741888_1066 (size=4906) 2024-12-02T22:23:52,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741888_1066 (size=4906) 2024-12-02T22:23:52,977 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf1/06d02d3164644c0fb5a4cb60a033650a 2024-12-02T22:23:52,994 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06d02d3164644c0fb5a4cb60a033650a 2024-12-02T22:23:53,016 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf2/cc547f083c9b47eb870e2cbb0e9e493d is 29, key is r1/cf2:/1733178232394/DeleteFamily/seqid=0 2024-12-02T22:23:53,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741889_1067 (size=4906) 2024-12-02T22:23:53,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741889_1067 (size=4906) 2024-12-02T22:23:53,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741889_1067 (size=4906) 2024-12-02T22:23:53,045 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf2/cc547f083c9b47eb870e2cbb0e9e493d 2024-12-02T22:23:53,054 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cc547f083c9b47eb870e2cbb0e9e493d 2024-12-02T22:23:53,056 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf1/06d02d3164644c0fb5a4cb60a033650a as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/06d02d3164644c0fb5a4cb60a033650a 2024-12-02T22:23:53,074 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06d02d3164644c0fb5a4cb60a033650a 2024-12-02T22:23:53,074 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/06d02d3164644c0fb5a4cb60a033650a, entries=1, sequenceid=12, filesize=4.8 K 2024-12-02T22:23:53,075 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf2/cc547f083c9b47eb870e2cbb0e9e493d as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/cc547f083c9b47eb870e2cbb0e9e493d 2024-12-02T22:23:53,089 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cc547f083c9b47eb870e2cbb0e9e493d 2024-12-02T22:23:53,089 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/cc547f083c9b47eb870e2cbb0e9e493d, entries=1, sequenceid=12, filesize=4.8 K 2024-12-02T22:23:53,093 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 5f517d31aff1d4768327a1ec4cf716a7 in 638ms, sequenceid=12, compaction requested=false 2024-12-02T22:23:53,093 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5f517d31aff1d4768327a1ec4cf716a7: 2024-12-02T22:23:53,096 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T22:23:53,098 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5f517d31aff1d4768327a1ec4cf716a7/cf1 is initiating major compaction (all files) 2024-12-02T22:23:53,099 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T22:23:53,099 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:53,099 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5f517d31aff1d4768327a1ec4cf716a7/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:53,100 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/1edaf893bf514749b6ec4d2253c197ad, hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/06d02d3164644c0fb5a4cb60a033650a] into tmpdir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp, totalSize=9.5 K 2024-12-02T22:23:53,101 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1edaf893bf514749b6ec4d2253c197ad, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733178231082 2024-12-02T22:23:53,108 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 06d02d3164644c0fb5a4cb60a033650a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-02T22:23:53,143 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5f517d31aff1d4768327a1ec4cf716a7#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:23:53,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741890_1068 (size=4626) 2024-12-02T22:23:53,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741890_1068 (size=4626) 2024-12-02T22:23:53,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741890_1068 (size=4626) 2024-12-02T22:23:53,286 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf1/02a4fb6bb66b4e288b5e80e49ed92dd0 as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/02a4fb6bb66b4e288b5e80e49ed92dd0 2024-12-02T22:23:53,314 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 5f517d31aff1d4768327a1ec4cf716a7/cf1 of 5f517d31aff1d4768327a1ec4cf716a7 into 02a4fb6bb66b4e288b5e80e49ed92dd0(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:23:53,314 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5f517d31aff1d4768327a1ec4cf716a7: 2024-12-02T22:23:53,314 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-02T22:23:53,314 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5f517d31aff1d4768327a1ec4cf716a7/cf2 is initiating major compaction (all files) 2024-12-02T22:23:53,314 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T22:23:53,314 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:23:53,315 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5f517d31aff1d4768327a1ec4cf716a7/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:53,315 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/cc547f083c9b47eb870e2cbb0e9e493d] into tmpdir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp, totalSize=4.8 K 2024-12-02T22:23:53,320 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cc547f083c9b47eb870e2cbb0e9e493d, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-02T22:23:53,346 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5f517d31aff1d4768327a1ec4cf716a7#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:23:53,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741891_1069 (size=4592) 2024-12-02T22:23:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741891_1069 (size=4592) 2024-12-02T22:23:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741891_1069 (size=4592) 2024-12-02T22:23:53,799 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/.tmp/cf2/bfa0fceb910e4bc9884f27c51e99564e as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/bfa0fceb910e4bc9884f27c51e99564e 2024-12-02T22:23:53,808 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 5f517d31aff1d4768327a1ec4cf716a7/cf2 of 5f517d31aff1d4768327a1ec4cf716a7 into bfa0fceb910e4bc9884f27c51e99564e(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:23:53,809 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5f517d31aff1d4768327a1ec4cf716a7: 2024-12-02T22:23:53,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=5f517d31aff1d4768327a1ec4cf716a7, source=7ae73638d47e,35511,1733178196345, destination=7ae73638d47e,38411,1733178196190, warming up region on 7ae73638d47e,38411,1733178196190 2024-12-02T22:23:53,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=5f517d31aff1d4768327a1ec4cf716a7, source=7ae73638d47e,35511,1733178196345, destination=7ae73638d47e,38411,1733178196190, running balancer 2024-12-02T22:23:53,828 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5f517d31aff1d4768327a1ec4cf716a7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:53,830 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:53,831 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf1 2024-12-02T22:23:53,831 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE 2024-12-02T22:23:53,833 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE 2024-12-02T22:23:53,845 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=CLOSING, regionLocation=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:53,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE because future has completed 2024-12-02T22:23:53,867 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/02a4fb6bb66b4e288b5e80e49ed92dd0 2024-12-02T22:23:53,874 INFO [StoreFileOpener-5f517d31aff1d4768327a1ec4cf716a7-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06d02d3164644c0fb5a4cb60a033650a 2024-12-02T22:23:53,874 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/06d02d3164644c0fb5a4cb60a033650a 2024-12-02T22:23:53,875 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:23:53,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345}] 2024-12-02T22:23:53,882 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/1edaf893bf514749b6ec4d2253c197ad 2024-12-02T22:23:53,883 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-02T22:23:53,883 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:53,886 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf2 2024-12-02T22:23:53,886 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:53,926 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/bfa0fceb910e4bc9884f27c51e99564e 2024-12-02T22:23:53,934 INFO [StoreFileOpener-5f517d31aff1d4768327a1ec4cf716a7-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cc547f083c9b47eb870e2cbb0e9e493d 2024-12-02T22:23:53,934 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/cc547f083c9b47eb870e2cbb0e9e493d 2024-12-02T22:23:53,934 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(1722): Closing 5f517d31aff1d4768327a1ec4cf716a7, disabling compactions & flushes 2024-12-02T22:23:53,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. after waiting 0 ms 2024-12-02T22:23:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:53,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38411 {}] regionserver.HRegion(1676): Region close journal for 5f517d31aff1d4768327a1ec4cf716a7: Waiting for close lock at 1733178233934Disabling compacts and flushes for region at 1733178233934Disabling writes for close at 1733178233935 (+1 ms)Writing region close event to WAL at 1733178233981 (+46 ms)Closed at 1733178233981 2024-12-02T22:23:54,057 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:23:54,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 5f517d31aff1d4768327a1ec4cf716a7, disabling compactions & flushes 2024-12-02T22:23:54,057 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. after waiting 0 ms 2024-12-02T22:23:54,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,065 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/1edaf893bf514749b6ec4d2253c197ad, hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/06d02d3164644c0fb5a4cb60a033650a] to archive 2024-12-02T22:23:54,072 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:23:54,100 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/1edaf893bf514749b6ec4d2253c197ad to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/1edaf893bf514749b6ec4d2253c197ad 2024-12-02T22:23:54,112 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/06d02d3164644c0fb5a4cb60a033650a to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/06d02d3164644c0fb5a4cb60a033650a 2024-12-02T22:23:54,153 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/cc547f083c9b47eb870e2cbb0e9e493d] to archive 2024-12-02T22:23:54,158 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:23:54,176 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/cc547f083c9b47eb870e2cbb0e9e493d to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/cc547f083c9b47eb870e2cbb0e9e493d 2024-12-02T22:23:54,203 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-02T22:23:54,204 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,204 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 5f517d31aff1d4768327a1ec4cf716a7: Waiting for close lock at 1733178234057Running coprocessor pre-close hooks at 1733178234057Disabling compacts and flushes for region at 1733178234057Disabling writes for close at 1733178234057Writing region close event to WAL at 1733178234188 (+131 ms)Running coprocessor post-close hooks at 1733178234204 (+16 ms)Closed at 1733178234204 2024-12-02T22:23:54,204 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 5f517d31aff1d4768327a1ec4cf716a7 move to 7ae73638d47e,38411,1733178196190 record at close sequenceid=12 2024-12-02T22:23:54,217 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=CLOSED 2024-12-02T22:23:54,217 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345 because future has completed 2024-12-02T22:23:54,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-02T22:23:54,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345 in 364 msec 2024-12-02T22:23:54,248 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE; state=CLOSED, location=7ae73638d47e,38411,1733178196190; forceNewPlan=false, retain=false 2024-12-02T22:23:54,399 INFO [7ae73638d47e:42475 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T22:23:54,399 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=OPENING, regionLocation=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:54,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE because future has completed 2024-12-02T22:23:54,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190}] 2024-12-02T22:23:54,562 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,562 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f517d31aff1d4768327a1ec4cf716a7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:54,563 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,563 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:54,563 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,563 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,567 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,569 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf1 2024-12-02T22:23:54,569 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:54,589 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/02a4fb6bb66b4e288b5e80e49ed92dd0 2024-12-02T22:23:54,590 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:54,591 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,593 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf2 2024-12-02T22:23:54,593 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:54,612 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/bfa0fceb910e4bc9884f27c51e99564e 2024-12-02T22:23:54,613 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:54,613 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,614 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,616 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,625 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,626 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,627 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T22:23:54,634 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,635 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 5f517d31aff1d4768327a1ec4cf716a7; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66451830, jitterRate=-0.009790569543838501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T22:23:54,635 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,636 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 5f517d31aff1d4768327a1ec4cf716a7: Running coprocessor pre-open hook at 1733178234563Writing region info on filesystem at 1733178234563Initializing all the Stores at 1733178234564 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178234564Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178234565 (+1 ms)Cleaning up temporary data from old regions at 1733178234626 (+61 ms)Running coprocessor post-open hooks at 1733178234635 (+9 ms)Region opened successfully at 1733178234636 (+1 ms) 2024-12-02T22:23:54,644 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., pid=12, masterSystemTime=1733178234557 2024-12-02T22:23:54,651 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,651 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,652 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=OPEN, openSeqNum=18, regionLocation=7ae73638d47e,38411,1733178196190 2024-12-02T22:23:54,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190 because future has completed 2024-12-02T22:23:54,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-02T22:23:54,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,38411,1733178196190 in 261 msec 2024-12-02T22:23:54,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, REOPEN/MOVE in 850 msec 2024-12-02T22:23:54,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T22:23:54,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T22:23:54,710 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 7ae73638d47e,38411,1733178196190: testing ***** 2024-12-02T22:23:54,710 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-02T22:23:54,713 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-02T22:23:54,715 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-02T22:23:54,726 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-02T22:23:54,729 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-02T22:23:54,745 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 618955912 }, "NonHeapMemoryUsage": { "committed": 174129152, "init": 7667712, "max": -1, "used": 168693176 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "7ae73638d47e", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 161, "ProcessCallTime_mean": 13, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 17, "ProcessCallTime_95th_percentile": 47, "ProcessCallTime_98th_percentile": 161, "ProcessCallTime_99th_percentile": 161, "ProcessCallTime_99.9th_percentile": 161, "ProcessCallTime_TimeRangeCount_0-1": 6, "ProcessCallTime_TimeRangeCount_1-3": 4, "ProcessCallTime_TimeRangeCount_3-10": 14, "ProcessCallTime_TimeRangeCount_10-30": 4, "ProcessCallTime_TimeRangeCount_30-100": 2, "ProcessCallTime_TimeRangeCount_100-300": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 4, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 1, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 2, "QueueCallTime_98th_percentile": 4, "QueueCallTime_99th_percentile": 4, "QueueCallTime_99.9th_percentile": 4, "QueueCallTime_TimeRangeCount_0-1": 28, "QueueCallTime_TimeRangeCount_1-3": 2, "QueueCallTime_TimeRangeCount_3-10": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 162, "TotalCallTime_mean": 13, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 11, "TotalCallTime_90th_percentile": 17, "TotalCallTime_95th_percentile": 47, "TotalCallTime_98th_percentile": 162, "TotalCallTime_99th_percentile": 162, "TotalCallTime_99.9th_percentile": 162, "TotalCallTime_TimeRangeCount_0-1": 5, "TotalCallTime_TimeRangeCount_1-3": 3, "TotalCallTime_TimeRangeCount_3-10": 15, "TotalCallTime_TimeRangeCount_10-30": 5, "TotalCallTime_TimeRangeCount_30-100": 2, "TotalCallTime_TimeRangeCount_100-300": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "7ae73638d47e", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55591", "tag.serverName": "7ae73638d47e,35511,1733178196345", "tag.clusterId": "9163252c-8e97-41ba-b6b3-4a5e364dbbc9", "tag.Context": "regionserver", "tag.Hostname": "7ae73638d47e", "regionCount": 0, "storeCount": 0, "hlogFileCount": 7, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733178196345, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 3, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 1, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 210, "MajorCompactionTime_max": 493, "MajorCompactionTime_mean": 351, "MajorCompactionTime_25th_percentile": 210, "MajorCompactionTime_median": 211, "MajorCompactionTime_75th_percentile": 374, "MajorCompactionTime_90th_percentile": 445, "MajorCompactionTime_95th_percentile": 469, "MajorCompactionTime_98th_percentile": 483, "MajorCompactionTime_99th_percentile": 488, "MajorCompactionTime_99.9th_percentile": 492, "MajorCompactionTime_TimeRangeCount_100-300": 2, "ScanTime_num_ops": 1, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 16, "Delete_max": 16, "Delete_mean": 16, "Delete_25th_percentile": 16, "Delete_median": 16, "Delete_75th_percentile": 16, "Delete_90th_percentile": 16, "Delete_95th_percentile": 16, "Delete_98th_percentile": 16, "Delete_99th_percentile": 16, "Delete_99.9th_percentile": 16, "Delete_TimeRangeCount_10-30": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 210, "CompactionTime_max": 493, "CompactionTime_mean": 351, "CompactionTime_25th_percentile": 210, "CompactionTime_median": 211, "CompactionTime_75th_percentile": 374, "CompactionTime_90th_percentile": 445, "CompactionTime_95th_percentile": 469, "CompactionTime_98th_percentile": 483, "CompactionTime_99th_percentile": 488, "CompactionTime_99.9th_percentile": 492, "CompactionTime_TimeRangeCount_100-300": 2, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 638, "FlushTime_max": 638, "FlushTime_mean": 638, "FlushTime_25th_percentile": 638, "FlushTime_median": 638, "FlushTime_75th_percentile": 638, "FlushTime_90th_percentile": 638, "FlushTime_95th_percentile": 638, "FlushTime_98th_percentile": 638, "FlushTime_99th_percentile": 638, "FlushTime_99.9th_percentile": 638, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 1, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-02T22:23:54,749 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42475 {}] master.MasterRpcServices(700): 7ae73638d47e,38411,1733178196190 reported a fatal error: ***** ABORTING region server 7ae73638d47e,38411,1733178196190: testing ***** 2024-12-02T22:23:54,773 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7ae73638d47e,38411,1733178196190' ***** 2024-12-02T22:23:54,773 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-02T22:23:54,775 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T22:23:54,775 INFO [RS:0;7ae73638d47e:38411 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-02T22:23:54,775 INFO [RS:0;7ae73638d47e:38411 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-02T22:23:54,775 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,776 INFO [regionserver/7ae73638d47e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T22:23:54,776 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(956): aborting server 7ae73638d47e,38411,1733178196190 2024-12-02T22:23:54,776 INFO [regionserver/7ae73638d47e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T22:23:54,776 INFO [RS:0;7ae73638d47e:38411 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T22:23:54,776 INFO [RS:0;7ae73638d47e:38411 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7ae73638d47e:38411. 2024-12-02T22:23:54,776 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f517d31aff1d4768327a1ec4cf716a7, disabling compactions & flushes 2024-12-02T22:23:54,776 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,776 DEBUG [RS:0;7ae73638d47e:38411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:23:54,777 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,777 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. after waiting 0 ms 2024-12-02T22:23:54,777 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,777 DEBUG [RS:0;7ae73638d47e:38411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:23:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:58720 deadline: 1733178294777, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7ae73638d47e port=38411 startCode=1733178196190. As of locationSeqNum=12. 2024-12-02T22:23:54,777 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T22:23:54,777 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T22:23:54,777 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T22:23:54,778 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T22:23:54,778 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,35511,1733178196345, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,35511,1733178196345, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7ae73638d47e port=38411 startCode=1733178196190. As of locationSeqNum=12. 2024-12-02T22:23:54,778 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,35511,1733178196345, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7ae73638d47e port=38411 startCode=1733178196190. As of locationSeqNum=12. 2024-12-02T22:23:54,778 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,35511,1733178196345, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7ae73638d47e port=38411 startCode=1733178196190. As of locationSeqNum=12. 2024-12-02T22:23:54,780 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T22:23:54,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7ae73638d47e,38411,1733178196190 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:54,783 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7ae73638d47e,38411,1733178196190 aborting 2024-12-02T22:23:54,783 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7ae73638d47e,38411,1733178196190 aborting 2024-12-02T22:23:54,783 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 from cache 2024-12-02T22:23:54,787 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T22:23:54,787 DEBUG [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5f517d31aff1d4768327a1ec4cf716a7=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.} 2024-12-02T22:23:54,788 DEBUG [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:54,789 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T22:23:54,789 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T22:23:54,789 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T22:23:54,789 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T22:23:54,789 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T22:23:54,789 INFO [regionserver/7ae73638d47e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T22:23:54,805 ERROR [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-02T22:23:54,805 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T22:23:54,812 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T22:23:54,813 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733178234788Running coprocessor pre-close hooks at 1733178234788Disabling compacts and flushes for region at 1733178234788Disabling writes for close at 1733178234789 (+1 ms)Writing region close event to WAL at 1733178234805 (+16 ms)Running coprocessor post-close hooks at 1733178234805Closed at 1733178234812 (+7 ms) 2024-12-02T22:23:54,813 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T22:23:54,833 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,833 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f517d31aff1d4768327a1ec4cf716a7: Waiting for close lock at 1733178234776Running coprocessor pre-close hooks at 1733178234776Disabling compacts and flushes for region at 1733178234776Disabling writes for close at 1733178234777 (+1 ms)Writing region close event to WAL at 1733178234833 (+56 ms)Running coprocessor post-close hooks at 1733178234833Closed at 1733178234833 2024-12-02T22:23:54,833 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:54,888 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:23:54,892 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1] 2024-12-02T22:23:54,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7ae73638d47e,38411,1733178196190 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:54,894 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7ae73638d47e,38411,1733178196190 aborting 2024-12-02T22:23:54,895 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7ae73638d47e,38411,1733178196190 aborting 2024-12-02T22:23:54,895 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 from cache 2024-12-02T22:23:54,988 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(976): stopping server 7ae73638d47e,38411,1733178196190; all regions closed. 2024-12-02T22:23:54,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741836_1012 (size=7323) 2024-12-02T22:23:54,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741836_1012 (size=7323) 2024-12-02T22:23:54,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741836_1012 (size=7323) 2024-12-02T22:23:55,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741834_1010 (size=2288) 2024-12-02T22:23:55,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741834_1010 (size=2288) 2024-12-02T22:23:55,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741834_1010 (size=2288) 2024-12-02T22:23:55,011 DEBUG [RS:0;7ae73638d47e:38411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:23:55,011 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T22:23:55,011 INFO [RS:0;7ae73638d47e:38411 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T22:23:55,012 INFO [RS:0;7ae73638d47e:38411 {}] hbase.ChoreService(370): Chore service for: regionserver/7ae73638d47e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T22:23:55,012 INFO [RS:0;7ae73638d47e:38411 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T22:23:55,013 INFO [RS:0;7ae73638d47e:38411 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38411 2024-12-02T22:23:55,017 INFO [regionserver/7ae73638d47e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T22:23:55,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T22:23:55,026 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7ae73638d47e,38411,1733178196190] 2024-12-02T22:23:55,026 INFO [RS:0;7ae73638d47e:38411 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T22:23:55,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7ae73638d47e,38411,1733178196190 2024-12-02T22:23:55,028 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7ae73638d47e,38411,1733178196190 already deleted, retry=false 2024-12-02T22:23:55,028 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 7ae73638d47e,38411,1733178196190 on 7ae73638d47e,42475,1733178195164 2024-12-02T22:23:55,036 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 7ae73638d47e,38411,1733178196190, splitWal=true, meta=true 2024-12-02T22:23:55,040 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 7ae73638d47e,38411,1733178196190, splitWal=true, meta=true 2024-12-02T22:23:55,042 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 7ae73638d47e,38411,1733178196190, splitWal=true, meta=true, isMeta: true 2024-12-02T22:23:55,045 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting 2024-12-02T22:23:55,046 INFO [PEWorker-2 {}] master.SplitWALManager(105): 7ae73638d47e,38411,1733178196190 WAL count=1, meta=true 2024-12-02T22:23:55,047 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 7ae73638d47e,38411,1733178196190 (carryingMeta=true) 7ae73638d47e,38411,1733178196190/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@15e3c76b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-02T22:23:55,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta}] 2024-12-02T22:23:55,057 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=7ae73638d47e,45899,1733178196419 2024-12-02T22:23:55,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, worker=7ae73638d47e,45899,1733178196419}] 2024-12-02T22:23:55,097 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:23:55,099 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1] 2024-12-02T22:23:55,101 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7ae73638d47e:38411 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7ae73638d47e/172.17.0.2:38411 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:55,102 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1, error=java.net.ConnectException: Call to address=7ae73638d47e:38411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7ae73638d47e/172.17.0.2:38411 2024-12-02T22:23:55,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-02T22:23:55,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 from cache 2024-12-02T22:23:55,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 7ae73638d47e:38411 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7ae73638d47e/172.17.0.2:38411 2024-12-02T22:23:55,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:23:55,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38411-0x1009d16841a0001, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:23:55,132 INFO [RS:0;7ae73638d47e:38411 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T22:23:55,132 INFO [RS:0;7ae73638d47e:38411 {}] regionserver.HRegionServer(1031): Exiting; stopping=7ae73638d47e,38411,1733178196190; zookeeper connection closed. 2024-12-02T22:23:55,132 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3e369b65 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3e369b65 2024-12-02T22:23:55,225 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T22:23:55,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53615, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T22:23:55,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-02T22:23:55,256 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, size=7.2 K (7323bytes) 2024-12-02T22:23:55,256 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta 2024-12-02T22:23:55,257 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta after 1ms 2024-12-02T22:23:55,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:55,302 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta took 47ms 2024-12-02T22:23:55,314 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-02T22:23:55,315 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta so closing down 2024-12-02T22:23:55,316 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:55,318 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta.temp 2024-12-02T22:23:55,360 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta.temp 2024-12-02T22:23:55,361 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:55,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741892_1070 (size=7071) 2024-12-02T22:23:55,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741892_1070 (size=7071) 2024-12-02T22:23:55,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741892_1070 (size=7071) 2024-12-02T22:23:55,391 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-02T22:23:55,393 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta.temp to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T22:23:55,393 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 89 ms; skipped=1; WAL=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-12-02T22:23:55,393 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, journal: Splitting hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, size=7.2 K (7323bytes) at 1733178235256Finishing writing output for hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta so closing down at 1733178235315 (+59 ms)Creating recovered edits writer path=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta.temp at 1733178235360 (+45 ms)3 split writer threads finished at 1733178235361 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733178235391 (+30 ms)Rename recovered edits hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta.temp to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733178235393 (+2 ms)Processed 16 edits across 1 Regions in 89 ms; skipped=1; WAL=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1733178235393 2024-12-02T22:23:55,393 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta 2024-12-02T22:23:55,395 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-02T22:23:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-02T22:23:55,406 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs 2024-12-02T22:23:55,407 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:23:55,409 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1] 2024-12-02T22:23:55,410 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 7ae73638d47e:38411 this server is in the failed servers list 2024-12-02T22:23:55,410 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=7ae73638d47e:38411 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 7ae73638d47e:38411 2024-12-02T22:23:55,411 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 7ae73638d47e:38411 2024-12-02T22:23:55,411 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 from cache 2024-12-02T22:23:55,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-02T22:23:55,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, worker=7ae73638d47e,45899,1733178196419 in 346 msec 2024-12-02T22:23:55,413 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=7ae73638d47e,45899,1733178196419 2024-12-02T22:23:55,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T22:23:55,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 7ae73638d47e%2C38411%2C1733178196190.meta.1733178200890.meta, worker=7ae73638d47e,45899,1733178196419 in 366 msec 2024-12-02T22:23:55,420 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting dir is empty, no logs to split. 2024-12-02T22:23:55,421 INFO [PEWorker-5 {}] master.SplitWALManager(105): 7ae73638d47e,38411,1733178196190 WAL count=0, meta=true 2024-12-02T22:23:55,421 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 7ae73638d47e,38411,1733178196190 WAL splitting is done? wals=0, meta=true 2024-12-02T22:23:55,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T22:23:55,426 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T22:23:55,428 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T22:23:55,578 DEBUG [7ae73638d47e:42475 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-02T22:23:55,579 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(204): Hosts are {7ae73638d47e=0} racks are {/default-rack=0} 2024-12-02T22:23:55,579 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T22:23:55,579 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T22:23:55,579 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T22:23:55,579 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T22:23:55,579 INFO [7ae73638d47e:42475 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T22:23:55,579 INFO [7ae73638d47e:42475 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T22:23:55,579 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T22:23:55,580 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:55,582 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7ae73638d47e,35511,1733178196345, state=OPENING 2024-12-02T22:23:55,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:55,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:55,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:55,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:55,584 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T22:23:55,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:55,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7ae73638d47e,35511,1733178196345}] 2024-12-02T22:23:55,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:55,743 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T22:23:55,743 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:55,744 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T22:23:55,747 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C35511%2C1733178196345.meta, suffix=.meta, logDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,35511,1733178196345, archiveDir=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs, maxLogs=32 2024-12-02T22:23:55,768 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,35511,1733178196345/7ae73638d47e%2C35511%2C1733178196345.meta.1733178235747.meta, exclude list is [], retry=0 2024-12-02T22:23:55,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:55,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:55,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:55,815 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,35511,1733178196345/7ae73638d47e%2C35511%2C1733178196345.meta.1733178235747.meta 2024-12-02T22:23:55,816 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:55,816 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:55,817 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T22:23:55,817 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T22:23:55,817 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T22:23:55,817 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T22:23:55,817 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:55,818 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T22:23:55,818 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T22:23:55,819 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T22:23:55,821 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T22:23:55,821 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:55,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:55,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T22:23:55,822 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T22:23:55,823 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:55,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:55,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T22:23:55,824 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T22:23:55,824 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:55,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:55,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T22:23:55,826 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T22:23:55,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:55,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:23:55,826 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T22:23:55,827 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740 2024-12-02T22:23:55,829 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740 2024-12-02T22:23:55,830 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T22:23:55,873 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:55,876 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T22:23:55,877 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-02T22:23:55,909 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/info/75189c06f36642b9a6e3a36d085e9ca1 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7./info:regioninfo/1733178234652/Put/seqid=0 2024-12-02T22:23:55,918 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:23:55,919 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,35511,1733178196345, seqNum=-1] 2024-12-02T22:23:55,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741894_1072 (size=11177) 2024-12-02T22:23:55,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741894_1072 (size=11177) 2024-12-02T22:23:55,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741894_1072 (size=11177) 2024-12-02T22:23:55,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:58720 deadline: 1733178295921, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 7ae73638d47e,35511,1733178196345 2024-12-02T22:23:55,929 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,35511,1733178196345, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,35511,1733178196345, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 7ae73638d47e,35511,1733178196345 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:23:55,929 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,35511,1733178196345, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 7ae73638d47e,35511,1733178196345 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:23:55,929 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,35511,1733178196345, seqNum=-1 from cache 2024-12-02T22:23:55,932 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/info/75189c06f36642b9a6e3a36d085e9ca1 2024-12-02T22:23:55,968 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/ns/d8437e6b34b64ea8bc1fd8c26f5f6b3d is 43, key is default/ns:d/1733178201411/Put/seqid=0 2024-12-02T22:23:55,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741895_1073 (size=5153) 2024-12-02T22:23:55,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741895_1073 (size=5153) 2024-12-02T22:23:56,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741895_1073 (size=5153) 2024-12-02T22:23:56,006 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/ns/d8437e6b34b64ea8bc1fd8c26f5f6b3d 2024-12-02T22:23:56,036 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/table/a80ac4b3ad1b4138a7e5e125246c5934 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733178230755/Put/seqid=0 2024-12-02T22:23:56,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741896_1074 (size=5431) 2024-12-02T22:23:56,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741896_1074 (size=5431) 2024-12-02T22:23:56,066 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/table/a80ac4b3ad1b4138a7e5e125246c5934 2024-12-02T22:23:56,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741896_1074 (size=5431) 2024-12-02T22:23:56,076 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/info/75189c06f36642b9a6e3a36d085e9ca1 as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/info/75189c06f36642b9a6e3a36d085e9ca1 2024-12-02T22:23:56,083 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/info/75189c06f36642b9a6e3a36d085e9ca1, entries=36, sequenceid=18, filesize=10.9 K 2024-12-02T22:23:56,084 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/ns/d8437e6b34b64ea8bc1fd8c26f5f6b3d as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/ns/d8437e6b34b64ea8bc1fd8c26f5f6b3d 2024-12-02T22:23:56,090 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/ns/d8437e6b34b64ea8bc1fd8c26f5f6b3d, entries=2, sequenceid=18, filesize=5.0 K 2024-12-02T22:23:56,091 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/table/a80ac4b3ad1b4138a7e5e125246c5934 as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/table/a80ac4b3ad1b4138a7e5e125246c5934 2024-12-02T22:23:56,097 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/table/a80ac4b3ad1b4138a7e5e125246c5934, entries=2, sequenceid=18, filesize=5.3 K 2024-12-02T22:23:56,097 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 221ms, sequenceid=18, compaction requested=false; wal=null 2024-12-02T22:23:56,098 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T22:23:56,099 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:23:56,105 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T22:23:56,105 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T22:23:56,106 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T22:23:56,108 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T22:23:56,117 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-02T22:23:56,119 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61768336, jitterRate=-0.07958006858825684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T22:23:56,119 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T22:23:56,120 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733178235818Writing region info on filesystem at 1733178235818Initializing all the Stores at 1733178235819 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178235819Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178235819Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178235819Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733178235819Obtaining lock to block concurrent updates at 1733178235877 (+58 ms)Preparing flush snapshotting stores in 1588230740 at 1733178235877Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733178235877Flushing stores of hbase:meta,,1.1588230740 at 1733178235877Flushing 1588230740/info: creating writer at 1733178235877Flushing 1588230740/info: appending metadata at 1733178235908 (+31 ms)Flushing 1588230740/info: closing flushed file at 1733178235908Flushing 1588230740/ns: creating writer at 1733178235943 (+35 ms)Flushing 1588230740/ns: appending metadata at 1733178235967 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1733178235967Flushing 1588230740/table: creating writer at 1733178236013 (+46 ms)Flushing 1588230740/table: appending metadata at 1733178236035 (+22 ms)Flushing 1588230740/table: closing flushed file at 1733178236035Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e38d65: reopening flushed file at 1733178236075 (+40 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@662dc5ee: reopening flushed file at 1733178236083 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64d7ecbe: reopening flushed file at 1733178236090 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 221ms, sequenceid=18, compaction requested=false; wal=null at 1733178236097 (+7 ms)Cleaning up temporary data from old regions at 1733178236105 (+8 ms)Running coprocessor post-open hooks at 1733178236119 (+14 ms)Region opened successfully at 1733178236120 (+1 ms) 2024-12-02T22:23:56,124 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733178235738 2024-12-02T22:23:56,128 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T22:23:56,128 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T22:23:56,128 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:56,129 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7ae73638d47e,35511,1733178196345, state=OPEN 2024-12-02T22:23:56,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:56,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:56,132 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:56,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:23:56,132 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:56,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:23:56,134 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:56,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-02T22:23:56,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7ae73638d47e,35511,1733178196345 in 550 msec 2024-12-02T22:23:56,143 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): 7ae73638d47e,38411,1733178196190 had 2 regions 2024-12-02T22:23:56,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-02T22:23:56,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 715 msec 2024-12-02T22:23:56,144 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 7ae73638d47e,38411,1733178196190, splitWal=true, meta=true, isMeta: false 2024-12-02T22:23:56,147 INFO [PEWorker-5 {}] master.SplitWALManager(105): 7ae73638d47e,38411,1733178196190 WAL count=1, meta=false 2024-12-02T22:23:56,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 7ae73638d47e%2C38411%2C1733178196190.1733178200029}] 2024-12-02T22:23:56,151 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=7ae73638d47e,45899,1733178196419 2024-12-02T22:23:56,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 7ae73638d47e%2C38411%2C1733178196190.1733178200029, worker=7ae73638d47e,45899,1733178196419}] 2024-12-02T22:23:56,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-02T22:23:56,322 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029, size=2.2 K (2288bytes) 2024-12-02T22:23:56,322 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029 2024-12-02T22:23:56,323 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029 after 1ms 2024-12-02T22:23:56,350 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:56,350 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029 took 28ms 2024-12-02T22:23:56,353 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 5f517d31aff1d4768327a1ec4cf716a7: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-02T22:23:56,353 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029 so closing down 2024-12-02T22:23:56,353 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:56,354 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:56,354 INFO [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-02T22:23:56,354 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029, journal: Splitting hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029, size=2.2 K (2288bytes) at 1733178236322Finishing writing output for hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029 so closing down at 1733178236353 (+31 ms)3 split writer threads finished at 1733178236354 (+1 ms)Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733178236354 2024-12-02T22:23:56,354 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029 2024-12-02T22:23:56,354 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-02T22:23:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-02T22:23:56,359 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting/7ae73638d47e%2C38411%2C1733178196190.1733178200029 to hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs 2024-12-02T22:23:56,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-02T22:23:56,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7ae73638d47e%2C38411%2C1733178196190.1733178200029, worker=7ae73638d47e,45899,1733178196419 in 207 msec 2024-12-02T22:23:56,363 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=7ae73638d47e,45899,1733178196419 2024-12-02T22:23:56,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-02T22:23:56,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 7ae73638d47e%2C38411%2C1733178196190.1733178200029, worker=7ae73638d47e,45899,1733178196419 in 217 msec 2024-12-02T22:23:56,368 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/WALs/7ae73638d47e,38411,1733178196190-splitting dir is empty, no logs to split. 2024-12-02T22:23:56,368 INFO [PEWorker-5 {}] master.SplitWALManager(105): 7ae73638d47e,38411,1733178196190 WAL count=0, meta=false 2024-12-02T22:23:56,368 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 7ae73638d47e,38411,1733178196190 WAL splitting is done? wals=0, meta=false 2024-12-02T22:23:56,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN}] 2024-12-02T22:23:56,372 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN 2024-12-02T22:23:56,373 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T22:23:56,524 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(204): Hosts are {7ae73638d47e=0} racks are {/default-rack=0} 2024-12-02T22:23:56,524 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T22:23:56,524 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T22:23:56,524 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T22:23:56,524 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T22:23:56,524 INFO [7ae73638d47e:42475 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T22:23:56,525 INFO [7ae73638d47e:42475 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T22:23:56,525 DEBUG [7ae73638d47e:42475 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T22:23:56,525 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=OPENING, regionLocation=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:56,527 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7ae73638d47e:38411 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7ae73638d47e/172.17.0.2:38411 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:23:56,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1, error=java.net.ConnectException: Call to address=7ae73638d47e:38411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7ae73638d47e/172.17.0.2:38411 2024-12-02T22:23:56,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-02T22:23:56,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,38411,1733178196190, seqNum=-1 from cache 2024-12-02T22:23:56,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 7ae73638d47e:38411 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7ae73638d47e/172.17.0.2:38411 2024-12-02T22:23:56,637 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:23:56,637 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,35511,1733178196345, seqNum=-1] 2024-12-02T22:23:56,638 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:23:56,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37039, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:23:56,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN because future has completed 2024-12-02T22:23:56,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345}] 2024-12-02T22:23:56,800 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:56,801 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f517d31aff1d4768327a1ec4cf716a7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:56,801 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,801 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:56,801 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,802 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,804 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,805 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf1 2024-12-02T22:23:56,805 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:56,827 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf1/02a4fb6bb66b4e288b5e80e49ed92dd0 2024-12-02T22:23:56,827 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:56,827 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,830 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f517d31aff1d4768327a1ec4cf716a7 columnFamilyName cf2 2024-12-02T22:23:56,830 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:56,854 DEBUG [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/cf2/bfa0fceb910e4bc9884f27c51e99564e 2024-12-02T22:23:56,854 INFO [StoreOpener-5f517d31aff1d4768327a1ec4cf716a7-1 {}] regionserver.HStore(327): Store=5f517d31aff1d4768327a1ec4cf716a7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:56,854 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,855 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,859 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,861 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,862 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,862 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T22:23:56,864 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,866 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 5f517d31aff1d4768327a1ec4cf716a7; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69528319, jitterRate=0.03605268895626068}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T22:23:56,866 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:23:56,867 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 5f517d31aff1d4768327a1ec4cf716a7: Running coprocessor pre-open hook at 1733178236802Writing region info on filesystem at 1733178236802Initializing all the Stores at 1733178236803 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178236803Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178236803Cleaning up temporary data from old regions at 1733178236862 (+59 ms)Running coprocessor post-open hooks at 1733178236866 (+4 ms)Region opened successfully at 1733178236867 (+1 ms) 2024-12-02T22:23:56,873 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., pid=21, masterSystemTime=1733178236796 2024-12-02T22:23:56,876 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:56,876 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:23:56,879 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5f517d31aff1d4768327a1ec4cf716a7, regionState=OPEN, openSeqNum=18, regionLocation=7ae73638d47e,35511,1733178196345 2024-12-02T22:23:56,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345 because future has completed 2024-12-02T22:23:56,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-02T22:23:56,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f517d31aff1d4768327a1ec4cf716a7, server=7ae73638d47e,35511,1733178196345 in 247 msec 2024-12-02T22:23:56,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-02T22:23:56,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5f517d31aff1d4768327a1ec4cf716a7, ASSIGN in 525 msec 2024-12-02T22:23:56,905 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 7ae73638d47e,38411,1733178196190 after splitting done 2024-12-02T22:23:56,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 7ae73638d47e,38411,1733178196190, splitWal=true, meta=true in 1.8750 sec 2024-12-02T22:23:56,940 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:23:56,946 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,35511,1733178196345, seqNum=-1] 2024-12-02T22:23:57,081 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 7ae73638d47e:38411 this server is in the failed servers list 2024-12-02T22:23:57,081 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=7ae73638d47e:38411 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 7ae73638d47e:38411 2024-12-02T22:23:57,081 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 7ae73638d47e:38411 2024-12-02T22:23:57,081 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,38411,1733178196190, seqNum=12 from cache 2024-12-02T22:23:57,296 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7., hostname=7ae73638d47e,35511,1733178196345, seqNum=18] 2024-12-02T22:23:57,317 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=399 (was 406), OpenFileDescriptor=1028 (was 1013) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=924 (was 909) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3791 (was 3969) 2024-12-02T22:23:57,320 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1028 is superior to 1024 2024-12-02T22:23:57,337 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=398, OpenFileDescriptor=1028, MaxFileDescriptor=1048576, SystemLoadAverage=924, ProcessCount=11, AvailableMemoryMB=3790 2024-12-02T22:23:57,337 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1028 is superior to 1024 2024-12-02T22:23:57,361 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:57,366 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:23:57,367 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:23:57,375 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-05478739, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-05478739, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:57,396 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-05478739/hregion-05478739.1733178237376, exclude list is [], retry=0 2024-12-02T22:23:57,400 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:57,401 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:57,409 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:57,453 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-05478739/hregion-05478739.1733178237376 2024-12-02T22:23:57,453 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:23:57,454 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => da425228f1c3b7392f5b3dca66345b49, NAME => 'testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:23:57,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741898_1076 (size=67) 2024-12-02T22:23:57,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741898_1076 (size=67) 2024-12-02T22:23:57,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741898_1076 (size=67) 2024-12-02T22:23:57,886 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:57,888 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,889 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName a 2024-12-02T22:23:57,889 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:57,890 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:57,890 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,892 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName b 2024-12-02T22:23:57,892 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:57,892 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:57,893 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,895 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName c 2024-12-02T22:23:57,895 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:57,895 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:57,895 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,896 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,897 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,898 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,898 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,899 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:57,900 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:57,904 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:23:57,904 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened da425228f1c3b7392f5b3dca66345b49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72877411, jitterRate=0.08595804870128632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:57,905 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for da425228f1c3b7392f5b3dca66345b49: Writing region info on filesystem at 1733178237886Initializing all the Stores at 1733178237887 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178237887Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178237887Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178237887Cleaning up temporary data from old regions at 1733178237898 (+11 ms)Region opened successfully at 1733178237905 (+7 ms) 2024-12-02T22:23:57,905 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing da425228f1c3b7392f5b3dca66345b49, disabling compactions & flushes 2024-12-02T22:23:57,905 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. 2024-12-02T22:23:57,905 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. 2024-12-02T22:23:57,905 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. after waiting 0 ms 2024-12-02T22:23:57,905 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. 2024-12-02T22:23:57,909 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. 2024-12-02T22:23:57,909 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for da425228f1c3b7392f5b3dca66345b49: Waiting for close lock at 1733178237905Disabling compacts and flushes for region at 1733178237905Disabling writes for close at 1733178237905Writing region close event to WAL at 1733178237909 (+4 ms)Closed at 1733178237909 2024-12-02T22:23:57,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741897_1075 (size=175) 2024-12-02T22:23:57,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741897_1075 (size=175) 2024-12-02T22:23:57,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741897_1075 (size=175) 2024-12-02T22:23:57,927 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:23:57,927 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-05478739:(num 1733178237376) 2024-12-02T22:23:57,927 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:57,929 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:57,949 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929, exclude list is [], retry=0 2024-12-02T22:23:57,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:57,954 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:57,954 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:58,006 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929 2024-12-02T22:23:58,006 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:23:58,006 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => da425228f1c3b7392f5b3dca66345b49, NAME => 'testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:58,007 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:58,007 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,007 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,008 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,009 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName a 2024-12-02T22:23:58,009 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:58,010 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:58,010 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,011 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName b 2024-12-02T22:23:58,011 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:58,012 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:58,012 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,013 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName c 2024-12-02T22:23:58,013 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:58,014 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:58,014 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,014 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,016 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,017 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,017 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,017 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:58,019 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:58,019 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened da425228f1c3b7392f5b3dca66345b49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63238724, jitterRate=-0.05766957998275757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:58,020 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for da425228f1c3b7392f5b3dca66345b49: Writing region info on filesystem at 1733178238007Initializing all the Stores at 1733178238008 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178238008Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178238008Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178238008Cleaning up temporary data from old regions at 1733178238017 (+9 ms)Region opened successfully at 1733178238020 (+3 ms) 2024-12-02T22:23:58,082 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing da425228f1c3b7392f5b3dca66345b49 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-02T22:23:58,116 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/a/6dcf6fd9bfc644c998ac3f8b908b845e is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733178238020/Put/seqid=0 2024-12-02T22:23:58,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741900_1078 (size=5958) 2024-12-02T22:23:58,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741900_1078 (size=5958) 2024-12-02T22:23:58,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741900_1078 (size=5958) 2024-12-02T22:23:58,213 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/a/6dcf6fd9bfc644c998ac3f8b908b845e 2024-12-02T22:23:58,269 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/b/7eaf7675d93f4692be5f489b29dce8db is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733178238050/Put/seqid=0 2024-12-02T22:23:58,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741901_1079 (size=5958) 2024-12-02T22:23:58,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741901_1079 (size=5958) 2024-12-02T22:23:58,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741901_1079 (size=5958) 2024-12-02T22:23:58,406 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/b/7eaf7675d93f4692be5f489b29dce8db 2024-12-02T22:23:58,462 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/c/445e519afff648f9aa545c929b0d3727 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733178238068/Put/seqid=0 2024-12-02T22:23:58,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741902_1080 (size=5958) 2024-12-02T22:23:58,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741902_1080 (size=5958) 2024-12-02T22:23:58,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741902_1080 (size=5958) 2024-12-02T22:23:58,579 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/c/445e519afff648f9aa545c929b0d3727 2024-12-02T22:23:58,604 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/a/6dcf6fd9bfc644c998ac3f8b908b845e as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/a/6dcf6fd9bfc644c998ac3f8b908b845e 2024-12-02T22:23:58,634 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/a/6dcf6fd9bfc644c998ac3f8b908b845e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T22:23:58,644 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/b/7eaf7675d93f4692be5f489b29dce8db as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/b/7eaf7675d93f4692be5f489b29dce8db 2024-12-02T22:23:58,677 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/b/7eaf7675d93f4692be5f489b29dce8db, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T22:23:58,689 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/c/445e519afff648f9aa545c929b0d3727 as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/c/445e519afff648f9aa545c929b0d3727 2024-12-02T22:23:58,709 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/c/445e519afff648f9aa545c929b0d3727, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T22:23:58,713 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for da425228f1c3b7392f5b3dca66345b49 in 631ms, sequenceid=33, compaction requested=false 2024-12-02T22:23:58,713 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for da425228f1c3b7392f5b3dca66345b49: 2024-12-02T22:23:58,713 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing da425228f1c3b7392f5b3dca66345b49, disabling compactions & flushes 2024-12-02T22:23:58,714 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. 2024-12-02T22:23:58,714 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. 2024-12-02T22:23:58,714 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. after waiting 0 ms 2024-12-02T22:23:58,714 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. 2024-12-02T22:23:58,724 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. 2024-12-02T22:23:58,724 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for da425228f1c3b7392f5b3dca66345b49: Waiting for close lock at 1733178238713Disabling compacts and flushes for region at 1733178238713Disabling writes for close at 1733178238714 (+1 ms)Writing region close event to WAL at 1733178238723 (+9 ms)Closed at 1733178238724 (+1 ms) 2024-12-02T22:23:58,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741899_1077 (size=6311) 2024-12-02T22:23:58,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741899_1077 (size=6311) 2024-12-02T22:23:58,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741899_1077 (size=6311) 2024-12-02T22:23:58,757 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/b/7eaf7675d93f4692be5f489b29dce8db to hdfs://localhost:38075/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/b/7eaf7675d93f4692be5f489b29dce8db 2024-12-02T22:23:58,808 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929, size=6.2 K (6311bytes) 2024-12-02T22:23:58,808 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929 2024-12-02T22:23:58,813 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929 after 5ms 2024-12-02T22:23:58,876 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:58,877 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929 took 68ms 2024-12-02T22:23:58,891 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929 so closing down 2024-12-02T22:23:58,891 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:23:58,913 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733178237929.temp 2024-12-02T22:23:58,956 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000003-wal.1733178237929.temp 2024-12-02T22:23:58,959 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:23:59,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741903_1081 (size=5575) 2024-12-02T22:23:59,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741903_1081 (size=5575) 2024-12-02T22:23:59,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741903_1081 (size=5575) 2024-12-02T22:23:59,061 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000003-wal.1733178237929.temp (wrote 30 edits, skipped 0 edits in 3 ms) 2024-12-02T22:23:59,088 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000003-wal.1733178237929.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000032 2024-12-02T22:23:59,089 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 205 ms; skipped=2; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-02T22:23:59,089 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929, journal: Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929, size=6.2 K (6311bytes) at 1733178238808Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929 so closing down at 1733178238891 (+83 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000003-wal.1733178237929.temp at 1733178238956 (+65 ms)3 split writer threads finished at 1733178238959 (+3 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000003-wal.1733178237929.temp (wrote 30 edits, skipped 0 edits in 3 ms) at 1733178239061 (+102 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000003-wal.1733178237929.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000032 at 1733178239088 (+27 ms)Processed 32 edits across 1 Regions in 205 ms; skipped=2; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733178239089 (+1 ms) 2024-12-02T22:23:59,128 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178237929 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178237929 2024-12-02T22:23:59,137 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000032 2024-12-02T22:23:59,137 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:23:59,153 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:23:59,173 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178239153, exclude list is [], retry=0 2024-12-02T22:23:59,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:23:59,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:23:59,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:23:59,256 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178239153 2024-12-02T22:23:59,256 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:23:59,256 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => da425228f1c3b7392f5b3dca66345b49, NAME => 'testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:23:59,256 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:23:59,257 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,257 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,260 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,261 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName a 2024-12-02T22:23:59,261 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:59,278 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/a/6dcf6fd9bfc644c998ac3f8b908b845e 2024-12-02T22:23:59,278 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:59,279 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,289 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName b 2024-12-02T22:23:59,289 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:59,292 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:59,293 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,295 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da425228f1c3b7392f5b3dca66345b49 columnFamilyName c 2024-12-02T22:23:59,295 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:23:59,315 DEBUG [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/c/445e519afff648f9aa545c929b0d3727 2024-12-02T22:23:59,315 INFO [StoreOpener-da425228f1c3b7392f5b3dca66345b49-1 {}] regionserver.HStore(327): Store=da425228f1c3b7392f5b3dca66345b49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:23:59,316 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,321 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,327 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,328 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000032 2024-12-02T22:23:59,379 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:23:59,384 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000032 2024-12-02T22:23:59,384 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing da425228f1c3b7392f5b3dca66345b49 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-02T22:23:59,415 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/b/9873853234464ab4a62c75460543ca79 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733178238050/Put/seqid=0 2024-12-02T22:23:59,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741905_1083 (size=5958) 2024-12-02T22:23:59,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741905_1083 (size=5958) 2024-12-02T22:23:59,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741905_1083 (size=5958) 2024-12-02T22:23:59,865 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/b/9873853234464ab4a62c75460543ca79 2024-12-02T22:23:59,893 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/.tmp/b/9873853234464ab4a62c75460543ca79 as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/b/9873853234464ab4a62c75460543ca79 2024-12-02T22:23:59,929 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/b/9873853234464ab4a62c75460543ca79, entries=10, sequenceid=32, filesize=5.8 K 2024-12-02T22:23:59,930 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for da425228f1c3b7392f5b3dca66345b49 in 545ms, sequenceid=32, compaction requested=false; wal=null 2024-12-02T22:23:59,931 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/0000000000000000032 2024-12-02T22:23:59,932 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,932 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,936 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:23:59,948 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for da425228f1c3b7392f5b3dca66345b49 2024-12-02T22:23:59,977 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/da425228f1c3b7392f5b3dca66345b49/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-02T22:23:59,989 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened da425228f1c3b7392f5b3dca66345b49; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74240816, jitterRate=0.10627436637878418}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:23:59,990 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for da425228f1c3b7392f5b3dca66345b49: Writing region info on filesystem at 1733178239257Initializing all the Stores at 1733178239259 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178239259Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178239259Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178239259Obtaining lock to block concurrent updates at 1733178239384 (+125 ms)Preparing flush snapshotting stores in da425228f1c3b7392f5b3dca66345b49 at 1733178239384Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733178239384Flushing stores of testReplayEditsWrittenViaHRegion,,1733178237362.da425228f1c3b7392f5b3dca66345b49. at 1733178239384Flushing da425228f1c3b7392f5b3dca66345b49/b: creating writer at 1733178239384Flushing da425228f1c3b7392f5b3dca66345b49/b: appending metadata at 1733178239415 (+31 ms)Flushing da425228f1c3b7392f5b3dca66345b49/b: closing flushed file at 1733178239415Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b0dabd8: reopening flushed file at 1733178239889 (+474 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for da425228f1c3b7392f5b3dca66345b49 in 545ms, sequenceid=32, compaction requested=false; wal=null at 1733178239930 (+41 ms)Cleaning up temporary data from old regions at 1733178239932 (+2 ms)Region opened successfully at 1733178239990 (+58 ms) 2024-12-02T22:24:00,048 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=405 (was 398) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:51474 [Waiting for operation #48] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:47586 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:34536 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:41370 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:44770 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:34560 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1098 (was 1028) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=906 (was 924), ProcessCount=11 (was 11), AvailableMemoryMB=3674 (was 3790) 2024-12-02T22:24:00,048 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1098 is superior to 1024 2024-12-02T22:24:00,113 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=405, OpenFileDescriptor=1100, MaxFileDescriptor=1048576, SystemLoadAverage=906, ProcessCount=11, AvailableMemoryMB=3673 2024-12-02T22:24:00,113 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1100 is superior to 1024 2024-12-02T22:24:00,144 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:24:00,153 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:24:00,154 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:24:00,159 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72582000, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-72582000, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:00,180 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72582000/hregion-72582000.1733178240160, exclude list is [], retry=0 2024-12-02T22:24:00,185 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:00,205 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:00,218 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:00,294 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72582000/hregion-72582000.1733178240160 2024-12-02T22:24:00,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:24:00,303 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4807f8ee5df40da869b50f1e324c7670, NAME => 'testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:24:00,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741907_1085 (size=68) 2024-12-02T22:24:00,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741907_1085 (size=68) 2024-12-02T22:24:00,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741907_1085 (size=68) 2024-12-02T22:24:00,803 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:00,811 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,814 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName a 2024-12-02T22:24:00,814 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:00,815 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:00,815 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,820 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName b 2024-12-02T22:24:00,820 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:00,821 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:00,821 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,825 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName c 2024-12-02T22:24:00,825 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:00,828 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:00,829 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,829 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,830 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,831 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,831 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,832 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:00,834 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:00,837 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:24:00,838 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4807f8ee5df40da869b50f1e324c7670; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70187939, jitterRate=0.04588179290294647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:00,839 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4807f8ee5df40da869b50f1e324c7670: Writing region info on filesystem at 1733178240804Initializing all the Stores at 1733178240811 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178240811Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178240811Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178240811Cleaning up temporary data from old regions at 1733178240831 (+20 ms)Region opened successfully at 1733178240839 (+8 ms) 2024-12-02T22:24:00,839 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4807f8ee5df40da869b50f1e324c7670, disabling compactions & flushes 2024-12-02T22:24:00,839 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:00,839 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:00,839 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. after waiting 0 ms 2024-12-02T22:24:00,839 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:00,840 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:00,840 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4807f8ee5df40da869b50f1e324c7670: Waiting for close lock at 1733178240839Disabling compacts and flushes for region at 1733178240839Disabling writes for close at 1733178240839Writing region close event to WAL at 1733178240840 (+1 ms)Closed at 1733178240840 2024-12-02T22:24:00,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741906_1084 (size=175) 2024-12-02T22:24:00,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741906_1084 (size=175) 2024-12-02T22:24:00,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741906_1084 (size=175) 2024-12-02T22:24:00,853 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:24:00,853 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72582000:(num 1733178240160) 2024-12-02T22:24:00,853 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:00,856 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:00,877 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856, exclude list is [], retry=0 2024-12-02T22:24:00,880 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:00,880 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:00,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:00,923 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856 2024-12-02T22:24:00,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:24:01,019 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4807f8ee5df40da869b50f1e324c7670, NAME => 'testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:01,023 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,023 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:01,023 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,023 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,026 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,028 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName a 2024-12-02T22:24:01,028 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:01,029 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:01,029 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,030 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName b 2024-12-02T22:24:01,030 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:01,035 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:01,035 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,036 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName c 2024-12-02T22:24:01,036 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:01,037 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:01,037 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,038 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,041 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,042 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,042 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,043 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:01,045 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,046 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4807f8ee5df40da869b50f1e324c7670; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64622427, jitterRate=-0.037050798535346985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:01,046 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:01,047 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4807f8ee5df40da869b50f1e324c7670: Running coprocessor pre-open hook at 1733178241023Writing region info on filesystem at 1733178241023Initializing all the Stores at 1733178241025 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178241025Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178241026 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178241026Cleaning up temporary data from old regions at 1733178241042 (+16 ms)Running coprocessor post-open hooks at 1733178241046 (+4 ms)Region opened successfully at 1733178241047 (+1 ms) 2024-12-02T22:24:01,081 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4807f8ee5df40da869b50f1e324c7670 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-02T22:24:01,085 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:01,320 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:24:01,539 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T22:24:01,693 INFO [master/7ae73638d47e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T22:24:01,693 INFO [master/7ae73638d47e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T22:24:02,087 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:03,088 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:04,089 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:05,090 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:05,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-02T22:24:05,685 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-02T22:24:05,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-02T22:24:05,686 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-02T22:24:05,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-02T22:24:05,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-02T22:24:06,091 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:06,525 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:24:07,092 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:08,093 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:09,093 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:10,094 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4807f8ee5df40da869b50f1e324c7670/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:10,095 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4807f8ee5df40da869b50f1e324c7670: 2024-12-02T22:24:10,096 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:10,131 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4807f8ee5df40da869b50f1e324c7670: 2024-12-02T22:24:10,131 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-02T22:24:10,131 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4807f8ee5df40da869b50f1e324c7670, disabling compactions & flushes 2024-12-02T22:24:10,131 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:10,131 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:10,131 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. after waiting 0 ms 2024-12-02T22:24:10,131 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:10,132 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:10,132 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. 2024-12-02T22:24:10,132 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4807f8ee5df40da869b50f1e324c7670: Waiting for close lock at 1733178250131Running coprocessor pre-close hooks at 1733178250131Disabling compacts and flushes for region at 1733178250131Disabling writes for close at 1733178250131Writing region close event to WAL at 1733178250132 (+1 ms)Running coprocessor post-close hooks at 1733178250132Closed at 1733178250132 2024-12-02T22:24:10,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741908_1086 (size=4236) 2024-12-02T22:24:10,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741908_1086 (size=4236) 2024-12-02T22:24:10,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741908_1086 (size=4236) 2024-12-02T22:24:10,233 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856, size=4.1 K (4236bytes) 2024-12-02T22:24:10,233 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856 2024-12-02T22:24:10,236 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856 after 3ms 2024-12-02T22:24:10,311 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:10,311 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856 took 78ms 2024-12-02T22:24:10,347 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856 so closing down 2024-12-02T22:24:10,347 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:24:10,373 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733178240856.temp 2024-12-02T22:24:10,460 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000004-wal.1733178240856.temp 2024-12-02T22:24:10,461 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:24:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741909_1087 (size=3245) 2024-12-02T22:24:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741909_1087 (size=3245) 2024-12-02T22:24:10,563 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000004-wal.1733178240856.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-02T22:24:10,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741909_1087 (size=3245) 2024-12-02T22:24:10,575 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000004-wal.1733178240856.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000026 2024-12-02T22:24:10,575 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 252 ms; skipped=3; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-02T22:24:10,575 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856, journal: Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856, size=4.1 K (4236bytes) at 1733178250233Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856 so closing down at 1733178250347 (+114 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000004-wal.1733178240856.temp at 1733178250460 (+113 ms)3 split writer threads finished at 1733178250461 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000004-wal.1733178240856.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733178250563 (+102 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000004-wal.1733178240856.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000026 at 1733178250575 (+12 ms)Processed 23 edits across 1 Regions in 252 ms; skipped=3; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733178250575 2024-12-02T22:24:10,600 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178240856 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178240856 2024-12-02T22:24:10,621 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000026 2024-12-02T22:24:10,621 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:10,652 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:10,680 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178250653, exclude list is [], retry=0 2024-12-02T22:24:10,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:10,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:10,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:10,777 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178250653 2024-12-02T22:24:10,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:24:10,781 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4807f8ee5df40da869b50f1e324c7670, NAME => 'testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:10,782 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,782 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:10,782 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,782 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,805 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,817 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName a 2024-12-02T22:24:10,817 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:10,828 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:10,829 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,841 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName b 2024-12-02T22:24:10,841 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:10,844 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:10,845 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,857 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4807f8ee5df40da869b50f1e324c7670 columnFamilyName c 2024-12-02T22:24:10,857 DEBUG [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:10,868 INFO [StoreOpener-4807f8ee5df40da869b50f1e324c7670-1 {}] regionserver.HStore(327): Store=4807f8ee5df40da869b50f1e324c7670/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:10,869 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,874 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:10,884 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000026 2024-12-02T22:24:10,955 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:10,963 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000026 2024-12-02T22:24:10,964 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4807f8ee5df40da869b50f1e324c7670 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-02T22:24:10,964 ERROR [Time-limited test {}] util.ResourceLeakDetector(337): LEAK: RefCnt.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information. Recent access records: Created at: org.apache.hadoop.hbase.nio.RefCnt.(RefCnt.java:59) org.apache.hadoop.hbase.nio.RefCnt.create(RefCnt.java:54) org.apache.hadoop.hbase.regionserver.MemStoreLABImpl.(MemStoreLABImpl.java:107) jdk.internal.reflect.GeneratedConstructorAccessor48.newInstance(Unknown Source) java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) org.apache.hadoop.hbase.util.ReflectionUtils.instantiate(ReflectionUtils.java:61) org.apache.hadoop.hbase.util.ReflectionUtils.instantiateWithCustomCtor(ReflectionUtils.java:49) org.apache.hadoop.hbase.regionserver.MemStoreLAB.newInstance(MemStoreLAB.java:116) org.apache.hadoop.hbase.regionserver.SegmentFactory.createMutableSegment(SegmentFactory.java:82) org.apache.hadoop.hbase.regionserver.AbstractMemStore.resetActive(AbstractMemStore.java:92) org.apache.hadoop.hbase.regionserver.DefaultMemStore.snapshot(DefaultMemStore.java:108) org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.prepare(HStore.java:1958) org.apache.hadoop.hbase.regionserver.HRegion.lambda$internalPrepareFlushCache$5(HRegion.java:2863) java.base/java.util.TreeMap.forEach(TreeMap.java:1282) org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2862) org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2735) org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testSequentialEditLogSeqNum(AbstractTestWALReplay.java:847) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:24:11,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/a/4ba411a91dbe47568e1c07ea4c599d0b is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733178250101/Put/seqid=0 2024-12-02T22:24:11,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741911_1089 (size=5523) 2024-12-02T22:24:11,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741911_1089 (size=5523) 2024-12-02T22:24:11,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741911_1089 (size=5523) 2024-12-02T22:24:11,108 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/a/4ba411a91dbe47568e1c07ea4c599d0b 2024-12-02T22:24:11,197 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/b/70d484dff1014e8097cbb51074990497 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733178250096/Put/seqid=0 2024-12-02T22:24:11,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741912_1090 (size=5524) 2024-12-02T22:24:11,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741912_1090 (size=5524) 2024-12-02T22:24:11,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741912_1090 (size=5524) 2024-12-02T22:24:11,281 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/b/70d484dff1014e8097cbb51074990497 2024-12-02T22:24:11,332 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/c/7fdef494e883484480fb390f3d192e66 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733178250098/Put/seqid=0 2024-12-02T22:24:11,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741913_1091 (size=5457) 2024-12-02T22:24:11,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741913_1091 (size=5457) 2024-12-02T22:24:11,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741913_1091 (size=5457) 2024-12-02T22:24:11,349 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/c/7fdef494e883484480fb390f3d192e66 2024-12-02T22:24:11,369 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/a/4ba411a91dbe47568e1c07ea4c599d0b as hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/a/4ba411a91dbe47568e1c07ea4c599d0b 2024-12-02T22:24:11,382 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/a/4ba411a91dbe47568e1c07ea4c599d0b, entries=7, sequenceid=26, filesize=5.4 K 2024-12-02T22:24:11,392 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/b/70d484dff1014e8097cbb51074990497 as hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/b/70d484dff1014e8097cbb51074990497 2024-12-02T22:24:11,431 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/b/70d484dff1014e8097cbb51074990497, entries=7, sequenceid=26, filesize=5.4 K 2024-12-02T22:24:11,434 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/.tmp/c/7fdef494e883484480fb390f3d192e66 as hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/c/7fdef494e883484480fb390f3d192e66 2024-12-02T22:24:11,447 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/c/7fdef494e883484480fb390f3d192e66, entries=6, sequenceid=26, filesize=5.3 K 2024-12-02T22:24:11,448 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 4807f8ee5df40da869b50f1e324c7670 in 484ms, sequenceid=26, compaction requested=false; wal=null 2024-12-02T22:24:11,449 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/0000000000000000026 2024-12-02T22:24:11,458 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:11,458 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:11,459 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:11,462 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:11,480 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsAfterAbortingFlush/4807f8ee5df40da869b50f1e324c7670/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-02T22:24:11,492 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4807f8ee5df40da869b50f1e324c7670; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70291418, jitterRate=0.047423750162124634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:11,492 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4807f8ee5df40da869b50f1e324c7670 2024-12-02T22:24:11,494 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4807f8ee5df40da869b50f1e324c7670: Running coprocessor pre-open hook at 1733178250783Writing region info on filesystem at 1733178250783Initializing all the Stores at 1733178250800 (+17 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178250801 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178250804 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178250804Obtaining lock to block concurrent updates at 1733178250964 (+160 ms)Preparing flush snapshotting stores in 4807f8ee5df40da869b50f1e324c7670 at 1733178250964Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733178250965 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733178240151.4807f8ee5df40da869b50f1e324c7670. at 1733178250965Flushing 4807f8ee5df40da869b50f1e324c7670/a: creating writer at 1733178250965Flushing 4807f8ee5df40da869b50f1e324c7670/a: appending metadata at 1733178251024 (+59 ms)Flushing 4807f8ee5df40da869b50f1e324c7670/a: closing flushed file at 1733178251024Flushing 4807f8ee5df40da869b50f1e324c7670/b: creating writer at 1733178251145 (+121 ms)Flushing 4807f8ee5df40da869b50f1e324c7670/b: appending metadata at 1733178251196 (+51 ms)Flushing 4807f8ee5df40da869b50f1e324c7670/b: closing flushed file at 1733178251196Flushing 4807f8ee5df40da869b50f1e324c7670/c: creating writer at 1733178251308 (+112 ms)Flushing 4807f8ee5df40da869b50f1e324c7670/c: appending metadata at 1733178251331 (+23 ms)Flushing 4807f8ee5df40da869b50f1e324c7670/c: closing flushed file at 1733178251331Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2665d674: reopening flushed file at 1733178251358 (+27 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79ef4d7a: reopening flushed file at 1733178251382 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6478218e: reopening flushed file at 1733178251432 (+50 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 4807f8ee5df40da869b50f1e324c7670 in 484ms, sequenceid=26, compaction requested=false; wal=null at 1733178251448 (+16 ms)Cleaning up temporary data from old regions at 1733178251458 (+10 ms)Running coprocessor post-open hooks at 1733178251492 (+34 ms)Region opened successfully at 1733178251493 (+1 ms) 2024-12-02T22:24:11,553 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=408 (was 405) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:57552 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:32896 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:54136 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:57568 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:32912 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:54116 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1174 (was 1100) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=906 (was 906), ProcessCount=11 (was 11), AvailableMemoryMB=4543 (was 3673) - AvailableMemoryMB LEAK? - 2024-12-02T22:24:11,553 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1174 is superior to 1024 2024-12-02T22:24:11,572 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=408, OpenFileDescriptor=1174, MaxFileDescriptor=1048576, SystemLoadAverage=906, ProcessCount=11, AvailableMemoryMB=4542 2024-12-02T22:24:11,572 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1174 is superior to 1024 2024-12-02T22:24:11,597 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:24:11,599 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:24:11,600 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:24:11,603 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79145894, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-79145894, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:11,641 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79145894/hregion-79145894.1733178251604, exclude list is [], retry=0 2024-12-02T22:24:11,657 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:11,665 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:11,667 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:11,720 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79145894/hregion-79145894.1733178251604 2024-12-02T22:24:11,720 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155)] 2024-12-02T22:24:11,721 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => acef103d2b78daca4306de39c339b8df, NAME => 'testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:24:11,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741915_1093 (size=61) 2024-12-02T22:24:11,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741915_1093 (size=61) 2024-12-02T22:24:11,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741915_1093 (size=61) 2024-12-02T22:24:12,149 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:12,161 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,171 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acef103d2b78daca4306de39c339b8df columnFamilyName a 2024-12-02T22:24:12,171 DEBUG [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:12,173 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.HStore(327): Store=acef103d2b78daca4306de39c339b8df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:12,176 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,180 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,182 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,184 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,184 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,190 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,212 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:24:12,213 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acef103d2b78daca4306de39c339b8df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70829842, jitterRate=0.055446892976760864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:24:12,214 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acef103d2b78daca4306de39c339b8df: Writing region info on filesystem at 1733178252150Initializing all the Stores at 1733178252156 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178252156Cleaning up temporary data from old regions at 1733178252184 (+28 ms)Region opened successfully at 1733178252214 (+30 ms) 2024-12-02T22:24:12,214 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing acef103d2b78daca4306de39c339b8df, disabling compactions & flushes 2024-12-02T22:24:12,214 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,215 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,215 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. after waiting 0 ms 2024-12-02T22:24:12,215 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,217 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,217 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for acef103d2b78daca4306de39c339b8df: Waiting for close lock at 1733178252214Disabling compacts and flushes for region at 1733178252214Disabling writes for close at 1733178252215 (+1 ms)Writing region close event to WAL at 1733178252217 (+2 ms)Closed at 1733178252217 2024-12-02T22:24:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741914_1092 (size=175) 2024-12-02T22:24:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741914_1092 (size=175) 2024-12-02T22:24:12,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741914_1092 (size=175) 2024-12-02T22:24:12,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:24:12,250 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79145894:(num 1733178251604) 2024-12-02T22:24:12,250 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:12,264 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:12,283 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265, exclude list is [], retry=0 2024-12-02T22:24:12,287 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:12,288 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:12,288 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:12,333 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265 2024-12-02T22:24:12,334 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:24:12,334 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => acef103d2b78daca4306de39c339b8df, NAME => 'testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:12,334 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:12,334 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,334 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,336 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,337 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acef103d2b78daca4306de39c339b8df columnFamilyName a 2024-12-02T22:24:12,337 DEBUG [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:12,338 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.HStore(327): Store=acef103d2b78daca4306de39c339b8df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:12,338 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,339 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,340 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,341 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,341 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,343 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:12,344 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acef103d2b78daca4306de39c339b8df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63117108, jitterRate=-0.059481799602508545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:24:12,345 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acef103d2b78daca4306de39c339b8df: Writing region info on filesystem at 1733178252334Initializing all the Stores at 1733178252336 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178252336Cleaning up temporary data from old regions at 1733178252341 (+5 ms)Region opened successfully at 1733178252345 (+4 ms) 2024-12-02T22:24:12,387 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing acef103d2b78daca4306de39c339b8df, disabling compactions & flushes 2024-12-02T22:24:12,387 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,387 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,387 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. after waiting 0 ms 2024-12-02T22:24:12,387 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,390 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,391 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. 2024-12-02T22:24:12,391 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for acef103d2b78daca4306de39c339b8df: Waiting for close lock at 1733178252387Disabling compacts and flushes for region at 1733178252387Disabling writes for close at 1733178252387Writing region close event to WAL at 1733178252390 (+3 ms)Closed at 1733178252391 (+1 ms) 2024-12-02T22:24:12,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741916_1094 (size=1795) 2024-12-02T22:24:12,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741916_1094 (size=1795) 2024-12-02T22:24:12,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741916_1094 (size=1795) 2024-12-02T22:24:12,452 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265, size=1.8 K (1795bytes) 2024-12-02T22:24:12,452 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265 2024-12-02T22:24:12,457 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265 after 5ms 2024-12-02T22:24:12,525 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:12,525 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265 took 73ms 2024-12-02T22:24:12,561 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265 so closing down 2024-12-02T22:24:12,573 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:24:12,593 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733178252265.temp 2024-12-02T22:24:12,668 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000003-wal.1733178252265.temp 2024-12-02T22:24:12,668 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:24:12,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741917_1095 (size=1795) 2024-12-02T22:24:12,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741917_1095 (size=1795) 2024-12-02T22:24:12,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741917_1095 (size=1795) 2024-12-02T22:24:12,740 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000003-wal.1733178252265.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-02T22:24:12,745 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000003-wal.1733178252265.temp to hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000012 2024-12-02T22:24:12,745 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 197 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-02T22:24:12,745 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265, journal: Splitting hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265, size=1.8 K (1795bytes) at 1733178252452Finishing writing output for hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265 so closing down at 1733178252562 (+110 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000003-wal.1733178252265.temp at 1733178252668 (+106 ms)3 split writer threads finished at 1733178252668Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000003-wal.1733178252265.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733178252740 (+72 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000003-wal.1733178252265.temp to hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000012 at 1733178252745 (+5 ms)Processed 10 edits across 1 Regions in 197 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733178252745 2024-12-02T22:24:12,747 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178252265 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178252265 2024-12-02T22:24:12,748 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000012 2024-12-02T22:24:12,791 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:13,663 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:13,683 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:13,714 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178253684, exclude list is [], retry=0 2024-12-02T22:24:13,722 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:13,723 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:13,724 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:13,790 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178253684 2024-12-02T22:24:13,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:24:13,801 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => acef103d2b78daca4306de39c339b8df, NAME => 'testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:13,801 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:13,801 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:13,801 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:13,808 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:13,829 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acef103d2b78daca4306de39c339b8df columnFamilyName a 2024-12-02T22:24:13,830 DEBUG [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:13,833 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.HStore(327): Store=acef103d2b78daca4306de39c339b8df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:13,833 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:13,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:13,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:13,856 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000012 2024-12-02T22:24:13,905 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:13,907 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000012 2024-12-02T22:24:13,908 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acef103d2b78daca4306de39c339b8df 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-02T22:24:13,998 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/.tmp/a/9e022bf336594974a813bb47cd428c11 is 79, key is testDatalossWhenInputError/a:x0/1733178252345/Put/seqid=0 2024-12-02T22:24:14,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741919_1097 (size=5808) 2024-12-02T22:24:14,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741919_1097 (size=5808) 2024-12-02T22:24:14,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741919_1097 (size=5808) 2024-12-02T22:24:14,100 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/.tmp/a/9e022bf336594974a813bb47cd428c11 2024-12-02T22:24:14,133 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/.tmp/a/9e022bf336594974a813bb47cd428c11 as hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/a/9e022bf336594974a813bb47cd428c11 2024-12-02T22:24:14,156 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/a/9e022bf336594974a813bb47cd428c11, entries=10, sequenceid=12, filesize=5.7 K 2024-12-02T22:24:14,157 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for acef103d2b78daca4306de39c339b8df in 250ms, sequenceid=12, compaction requested=false; wal=null 2024-12-02T22:24:14,158 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/0000000000000000012 2024-12-02T22:24:14,160 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,160 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,164 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,192 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-02T22:24:14,198 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acef103d2b78daca4306de39c339b8df; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63630609, jitterRate=-0.05183003842830658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:24:14,199 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acef103d2b78daca4306de39c339b8df: Writing region info on filesystem at 1733178253801Initializing all the Stores at 1733178253807 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178253808 (+1 ms)Obtaining lock to block concurrent updates at 1733178253908 (+100 ms)Preparing flush snapshotting stores in acef103d2b78daca4306de39c339b8df at 1733178253908Finished memstore snapshotting testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733178253908Flushing stores of testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df. at 1733178253908Flushing acef103d2b78daca4306de39c339b8df/a: creating writer at 1733178253908Flushing acef103d2b78daca4306de39c339b8df/a: appending metadata at 1733178253997 (+89 ms)Flushing acef103d2b78daca4306de39c339b8df/a: closing flushed file at 1733178253997Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bc2a18a: reopening flushed file at 1733178254126 (+129 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for acef103d2b78daca4306de39c339b8df in 250ms, sequenceid=12, compaction requested=false; wal=null at 1733178254157 (+31 ms)Cleaning up temporary data from old regions at 1733178254160 (+3 ms)Region opened successfully at 1733178254199 (+39 ms) 2024-12-02T22:24:14,204 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => acef103d2b78daca4306de39c339b8df, NAME => 'testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:14,204 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733178251597.acef103d2b78daca4306de39c339b8df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:14,204 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,204 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T22:24:14,204 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,212 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,228 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acef103d2b78daca4306de39c339b8df columnFamilyName a 2024-12-02T22:24:14,228 DEBUG [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:14,256 DEBUG [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/a/9e022bf336594974a813bb47cd428c11 2024-12-02T22:24:14,256 INFO [StoreOpener-acef103d2b78daca4306de39c339b8df-1 {}] regionserver.HStore(327): Store=acef103d2b78daca4306de39c339b8df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:14,256 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,260 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,262 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,262 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,262 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,265 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acef103d2b78daca4306de39c339b8df 2024-12-02T22:24:14,268 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testDatalossWhenInputError/acef103d2b78daca4306de39c339b8df/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-02T22:24:14,269 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acef103d2b78daca4306de39c339b8df; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68504436, jitterRate=0.02079564332962036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T22:24:14,269 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acef103d2b78daca4306de39c339b8df: Writing region info on filesystem at 1733178254205Initializing all the Stores at 1733178254206 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178254206Cleaning up temporary data from old regions at 1733178254262 (+56 ms)Region opened successfully at 1733178254269 (+7 ms) 2024-12-02T22:24:14,307 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=409 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:32982 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:54136 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:32912 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:57658 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-981501226-172.17.0.2-1733178187608:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:54214 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1258 (was 1174) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 906) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4341 (was 4542) 2024-12-02T22:24:14,307 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1258 is superior to 1024 2024-12-02T22:24:14,330 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=409, OpenFileDescriptor=1258, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4339 2024-12-02T22:24:14,330 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1258 is superior to 1024 2024-12-02T22:24:14,357 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:24:14,365 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:24:14,366 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:24:14,370 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16999773, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-16999773, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:14,390 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16999773/hregion-16999773.1733178254370, exclude list is [], retry=0 2024-12-02T22:24:14,409 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:14,413 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:14,417 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:14,554 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16999773/hregion-16999773.1733178254370 2024-12-02T22:24:14,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:24:14,555 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1126be5b68629f3e93b7a7a90c6f7944, NAME => 'testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:24:14,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741921_1099 (size=63) 2024-12-02T22:24:14,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741921_1099 (size=63) 2024-12-02T22:24:14,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741921_1099 (size=63) 2024-12-02T22:24:14,669 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:14,674 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,676 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName a 2024-12-02T22:24:14,676 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:14,677 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:14,678 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,707 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName b 2024-12-02T22:24:14,707 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:14,715 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:14,716 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,718 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName c 2024-12-02T22:24:14,719 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:14,723 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:14,727 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,732 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,733 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,737 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,737 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,742 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:14,753 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,755 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:24:14,756 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1126be5b68629f3e93b7a7a90c6f7944; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69419875, jitterRate=0.034436747431755066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:14,757 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1126be5b68629f3e93b7a7a90c6f7944: Writing region info on filesystem at 1733178254669Initializing all the Stores at 1733178254670 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178254670Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178254674 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178254674Cleaning up temporary data from old regions at 1733178254737 (+63 ms)Region opened successfully at 1733178254757 (+20 ms) 2024-12-02T22:24:14,757 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1126be5b68629f3e93b7a7a90c6f7944, disabling compactions & flushes 2024-12-02T22:24:14,757 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:14,757 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:14,757 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. after waiting 0 ms 2024-12-02T22:24:14,757 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:14,761 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:14,761 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1126be5b68629f3e93b7a7a90c6f7944: Waiting for close lock at 1733178254757Disabling compacts and flushes for region at 1733178254757Disabling writes for close at 1733178254757Writing region close event to WAL at 1733178254761 (+4 ms)Closed at 1733178254761 2024-12-02T22:24:14,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741920_1098 (size=175) 2024-12-02T22:24:14,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741920_1098 (size=175) 2024-12-02T22:24:14,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741920_1098 (size=175) 2024-12-02T22:24:14,783 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:24:14,783 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16999773:(num 1733178254370) 2024-12-02T22:24:14,784 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:14,794 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:14,826 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795, exclude list is [], retry=0 2024-12-02T22:24:14,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:14,830 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:14,830 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:14,884 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 2024-12-02T22:24:14,889 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:24:14,889 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1126be5b68629f3e93b7a7a90c6f7944, NAME => 'testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:14,889 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:14,889 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,889 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,918 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,919 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName a 2024-12-02T22:24:14,920 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:14,927 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:14,928 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,931 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName b 2024-12-02T22:24:14,931 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:14,931 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:14,932 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,933 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName c 2024-12-02T22:24:14,933 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:14,934 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:14,934 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,935 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,936 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,937 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,937 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,938 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:14,944 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:14,945 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1126be5b68629f3e93b7a7a90c6f7944; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61956360, jitterRate=-0.07677829265594482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:14,948 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1126be5b68629f3e93b7a7a90c6f7944: Writing region info on filesystem at 1733178254889Initializing all the Stores at 1733178254904 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178254904Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178254918 (+14 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178254918Cleaning up temporary data from old regions at 1733178254937 (+19 ms)Region opened successfully at 1733178254946 (+9 ms) 2024-12-02T22:24:14,959 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733178254958/Put/seqid=0 2024-12-02T22:24:14,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741923_1101 (size=4875) 2024-12-02T22:24:15,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741923_1101 (size=4875) 2024-12-02T22:24:15,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741923_1101 (size=4875) 2024-12-02T22:24:15,013 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733178255013/Put/seqid=0 2024-12-02T22:24:15,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741924_1102 (size=4875) 2024-12-02T22:24:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741924_1102 (size=4875) 2024-12-02T22:24:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741924_1102 (size=4875) 2024-12-02T22:24:15,086 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733178255085/Put/seqid=0 2024-12-02T22:24:15,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741925_1103 (size=4875) 2024-12-02T22:24:15,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741925_1103 (size=4875) 2024-12-02T22:24:15,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741925_1103 (size=4875) 2024-12-02T22:24:15,110 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 1126be5b68629f3e93b7a7a90c6f7944/a 2024-12-02T22:24:15,117 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-02T22:24:15,118 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T22:24:15,118 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 1126be5b68629f3e93b7a7a90c6f7944/a 2024-12-02T22:24:15,124 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-02T22:24:15,124 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T22:24:15,124 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 1126be5b68629f3e93b7a7a90c6f7944/a 2024-12-02T22:24:15,129 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-02T22:24:15,129 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T22:24:15,129 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1126be5b68629f3e93b7a7a90c6f7944 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-02T22:24:15,169 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/.tmp/a/0b90ce2b452a4337af23644157d81153 is 55, key is testCompactedBulkLoadedFiles/a:a/1733178254949/Put/seqid=0 2024-12-02T22:24:15,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741926_1104 (size=5107) 2024-12-02T22:24:15,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741926_1104 (size=5107) 2024-12-02T22:24:15,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741926_1104 (size=5107) 2024-12-02T22:24:15,199 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/.tmp/a/0b90ce2b452a4337af23644157d81153 2024-12-02T22:24:15,206 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/.tmp/a/0b90ce2b452a4337af23644157d81153 as hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/0b90ce2b452a4337af23644157d81153 2024-12-02T22:24:15,214 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/0b90ce2b452a4337af23644157d81153, entries=1, sequenceid=4, filesize=5.0 K 2024-12-02T22:24:15,215 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 1126be5b68629f3e93b7a7a90c6f7944 in 86ms, sequenceid=4, compaction requested=false 2024-12-02T22:24:15,215 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1126be5b68629f3e93b7a7a90c6f7944: 2024-12-02T22:24:15,217 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_ 2024-12-02T22:24:15,232 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_ 2024-12-02T22:24:15,234 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_ 2024-12-02T22:24:15,236 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile0 into 1126be5b68629f3e93b7a7a90c6f7944/a as hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_ - updating store file list. 2024-12-02T22:24:15,262 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 620d78e9387d42dc8eb65dad2819c26f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T22:24:15,262 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_ into 1126be5b68629f3e93b7a7a90c6f7944/a 2024-12-02T22:24:15,262 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile0 into 1126be5b68629f3e93b7a7a90c6f7944/a (new location: hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_) 2024-12-02T22:24:15,273 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile1 into 1126be5b68629f3e93b7a7a90c6f7944/a as hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_ - updating store file list. 2024-12-02T22:24:15,287 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T22:24:15,288 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_ into 1126be5b68629f3e93b7a7a90c6f7944/a 2024-12-02T22:24:15,288 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile1 into 1126be5b68629f3e93b7a7a90c6f7944/a (new location: hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_) 2024-12-02T22:24:15,294 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile2 into 1126be5b68629f3e93b7a7a90c6f7944/a as hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_ - updating store file list. 2024-12-02T22:24:15,302 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 52b917da83134b1d9d8bf3df06da093d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T22:24:15,302 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_ into 1126be5b68629f3e93b7a7a90c6f7944/a 2024-12-02T22:24:15,302 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38075/hbase/testCompactedBulkLoadedFiles/hfile2 into 1126be5b68629f3e93b7a7a90c6f7944/a (new location: hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_) 2024-12-02T22:24:15,365 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:24:15,365 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1126be5b68629f3e93b7a7a90c6f7944/a is initiating major compaction (all files) 2024-12-02T22:24:15,365 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1126be5b68629f3e93b7a7a90c6f7944/a in testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:15,366 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/0b90ce2b452a4337af23644157d81153, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_] into tmpdir=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/.tmp, totalSize=19.3 K 2024-12-02T22:24:15,370 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0b90ce2b452a4337af23644157d81153, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733178254949 2024-12-02T22:24:15,375 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 620d78e9387d42dc8eb65dad2819c26f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T22:24:15,377 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T22:24:15,385 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 52b917da83134b1d9d8bf3df06da093d_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T22:24:15,439 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/.tmp/a/a258de78bbd547cfa9bc3bfdb69cf342 is 55, key is testCompactedBulkLoadedFiles/a:a/1733178254949/Put/seqid=0 2024-12-02T22:24:15,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741927_1105 (size=6154) 2024-12-02T22:24:15,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741927_1105 (size=6154) 2024-12-02T22:24:15,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741927_1105 (size=6154) 2024-12-02T22:24:15,593 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/.tmp/a/a258de78bbd547cfa9bc3bfdb69cf342 as hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/a258de78bbd547cfa9bc3bfdb69cf342 2024-12-02T22:24:15,637 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:24:15,676 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 1126be5b68629f3e93b7a7a90c6f7944/a of 1126be5b68629f3e93b7a7a90c6f7944 into a258de78bbd547cfa9bc3bfdb69cf342(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:24:15,677 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1126be5b68629f3e93b7a7a90c6f7944: 2024-12-02T22:24:15,677 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-02T22:24:15,677 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-02T22:24:15,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-02T22:24:15,734 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-02T22:24:15,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-02T22:24:15,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-02T22:24:15,763 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795, size=0 (0bytes) 2024-12-02T22:24:15,763 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 might be still open, length is 0 2024-12-02T22:24:15,763 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 2024-12-02T22:24:15,765 WARN [IPC Server handler 0 on default port 38075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-02T22:24:15,767 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 after 4ms 2024-12-02T22:24:17,208 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:57678 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34817:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57678 dst: /127.0.0.1:34817 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34817 remote=/127.0.0.1:57678]. Total timeout mills is 60000, 58466 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:17,208 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:33014 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37315:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33014 dst: /127.0.0.1:37315 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:17,208 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:54238 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:40725:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54238 dst: /127.0.0.1:40725 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:17,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741922_1106 (size=1794) 2024-12-02T22:24:19,768 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 after 4005ms 2024-12-02T22:24:19,800 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:19,800 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 took 4038ms 2024-12-02T22:24:19,803 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795; continuing. 2024-12-02T22:24:19,803 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 so closing down 2024-12-02T22:24:19,803 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:24:19,804 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733178254795.temp 2024-12-02T22:24:19,841 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000003-wal.1733178254795.temp 2024-12-02T22:24:19,841 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:24:19,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741928_1107 (size=800) 2024-12-02T22:24:19,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741928_1107 (size=800) 2024-12-02T22:24:19,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741928_1107 (size=800) 2024-12-02T22:24:20,260 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000003-wal.1733178254795.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-02T22:24:20,265 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000003-wal.1733178254795.temp to hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000008 2024-12-02T22:24:20,266 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 465 ms; skipped=3; WAL=hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T22:24:20,266 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795, journal: Splitting hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795, size=0 (0bytes) at 1733178255763Finishing writing output for hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 so closing down at 1733178259803 (+4040 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000003-wal.1733178254795.temp at 1733178259841 (+38 ms)3 split writer threads finished at 1733178259841Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000003-wal.1733178254795.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733178260261 (+420 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000003-wal.1733178254795.temp to hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000008 at 1733178260265 (+4 ms)Processed 5 edits across 1 Regions in 465 ms; skipped=3; WAL=hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795, size=0, length=0, corrupted=false, cancelled=false at 1733178260266 (+1 ms) 2024-12-02T22:24:20,268 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178254795 2024-12-02T22:24:20,273 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000008 2024-12-02T22:24:20,273 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:20,276 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:20,313 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178260276, exclude list is [], retry=0 2024-12-02T22:24:20,321 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:20,321 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:20,322 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:20,421 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178260276 2024-12-02T22:24:20,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:24:20,425 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1126be5b68629f3e93b7a7a90c6f7944, NAME => 'testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:20,425 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:20,425 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,425 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,432 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,437 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName a 2024-12-02T22:24:20,437 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:20,484 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/0b90ce2b452a4337af23644157d81153 2024-12-02T22:24:20,488 DEBUG [StoreFileOpener-1126be5b68629f3e93b7a7a90c6f7944-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 52b917da83134b1d9d8bf3df06da093d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T22:24:20,492 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_ 2024-12-02T22:24:20,515 DEBUG [StoreFileOpener-1126be5b68629f3e93b7a7a90c6f7944-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 620d78e9387d42dc8eb65dad2819c26f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T22:24:20,519 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_ 2024-12-02T22:24:20,526 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/a258de78bbd547cfa9bc3bfdb69cf342 2024-12-02T22:24:20,529 DEBUG [StoreFileOpener-1126be5b68629f3e93b7a7a90c6f7944-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T22:24:20,529 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_ 2024-12-02T22:24:20,529 WARN [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/0b90ce2b452a4337af23644157d81153 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6f903938 2024-12-02T22:24:20,529 WARN [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6f903938 2024-12-02T22:24:20,529 WARN [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6f903938 2024-12-02T22:24:20,530 WARN [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6f903938 2024-12-02T22:24:20,530 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/0b90ce2b452a4337af23644157d81153, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_] to archive 2024-12-02T22:24:20,531 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:24:20,533 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/0b90ce2b452a4337af23644157d81153 to hdfs://localhost:38075/hbase/archive/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/0b90ce2b452a4337af23644157d81153 2024-12-02T22:24:20,535 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_ to hdfs://localhost:38075/hbase/archive/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/52b917da83134b1d9d8bf3df06da093d_SeqId_4_ 2024-12-02T22:24:20,536 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_ to hdfs://localhost:38075/hbase/archive/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/620d78e9387d42dc8eb65dad2819c26f_SeqId_4_ 2024-12-02T22:24:20,538 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_ to hdfs://localhost:38075/hbase/archive/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/a/c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_ 2024-12-02T22:24:20,538 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:20,538 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,539 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName b 2024-12-02T22:24:20,539 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:20,540 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:20,540 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,541 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126be5b68629f3e93b7a7a90c6f7944 columnFamilyName c 2024-12-02T22:24:20,541 DEBUG [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:20,545 INFO [StoreOpener-1126be5b68629f3e93b7a7a90c6f7944-1 {}] regionserver.HStore(327): Store=1126be5b68629f3e93b7a7a90c6f7944/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:20,545 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,546 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,548 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,548 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000008 2024-12-02T22:24:20,630 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:20,632 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 1126be5b68629f3e93b7a7a90c6f7944 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "1126be5b68629f3e93b7a7a90c6f7944" family_name: "a" compaction_input: "0b90ce2b452a4337af23644157d81153" compaction_input: "620d78e9387d42dc8eb65dad2819c26f_SeqId_4_" compaction_input: "c34ffe3f06eb4496be5b3a53538278ab_SeqId_4_" compaction_input: "52b917da83134b1d9d8bf3df06da093d_SeqId_4_" compaction_output: "a258de78bbd547cfa9bc3bfdb69cf342" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-02T22:24:20,633 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-02T22:24:20,633 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000008 2024-12-02T22:24:20,639 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/0000000000000000008 2024-12-02T22:24:20,640 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,640 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,641 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:20,650 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1126be5b68629f3e93b7a7a90c6f7944 2024-12-02T22:24:20,666 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testCompactedBulkLoadedFiles/1126be5b68629f3e93b7a7a90c6f7944/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-02T22:24:20,667 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1126be5b68629f3e93b7a7a90c6f7944; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63818235, jitterRate=-0.04903419315814972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:20,668 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1126be5b68629f3e93b7a7a90c6f7944: Writing region info on filesystem at 1733178260425Initializing all the Stores at 1733178260427 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178260427Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178260427Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178260427Cleaning up temporary data from old regions at 1733178260640 (+213 ms)Region opened successfully at 1733178260667 (+27 ms) 2024-12-02T22:24:20,675 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1126be5b68629f3e93b7a7a90c6f7944, disabling compactions & flushes 2024-12-02T22:24:20,675 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:20,675 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:20,675 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. after waiting 0 ms 2024-12-02T22:24:20,675 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:20,688 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733178254359.1126be5b68629f3e93b7a7a90c6f7944. 2024-12-02T22:24:20,688 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1126be5b68629f3e93b7a7a90c6f7944: Waiting for close lock at 1733178260675Disabling compacts and flushes for region at 1733178260675Disabling writes for close at 1733178260675Writing region close event to WAL at 1733178260688 (+13 ms)Closed at 1733178260688 2024-12-02T22:24:20,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741929_1108 (size=175) 2024-12-02T22:24:20,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741929_1108 (size=175) 2024-12-02T22:24:20,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741929_1108 (size=175) 2024-12-02T22:24:20,735 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:24:20,735 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733178260276) 2024-12-02T22:24:20,761 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=422 (was 409) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2104735759_22 at /127.0.0.1:47306 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:38075 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2104735759_22 at /127.0.0.1:43152 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2104735759_22 at /127.0.0.1:47002 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:38075 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1338 (was 1258) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=933 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=3944 (was 4339) 2024-12-02T22:24:20,762 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1338 is superior to 1024 2024-12-02T22:24:20,784 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=422, OpenFileDescriptor=1340, MaxFileDescriptor=1048576, SystemLoadAverage=933, ProcessCount=11, AvailableMemoryMB=3943 2024-12-02T22:24:20,784 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1340 is superior to 1024 2024-12-02T22:24:20,806 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:24:20,808 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:24:20,809 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:24:20,813 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-31902826, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/hregion-31902826, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:20,831 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-31902826/hregion-31902826.1733178260813, exclude list is [], retry=0 2024-12-02T22:24:20,836 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:20,837 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:20,841 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:21,445 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-31902826/hregion-31902826.1733178260813 2024-12-02T22:24:21,446 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:24:21,446 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 39c3fea49fafaa066e452b3008974b5a, NAME => 'testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38075/hbase 2024-12-02T22:24:21,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741931_1110 (size=67) 2024-12-02T22:24:21,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741931_1110 (size=67) 2024-12-02T22:24:21,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741931_1110 (size=67) 2024-12-02T22:24:21,470 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:21,472 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,474 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName a 2024-12-02T22:24:21,474 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:21,474 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:21,474 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,475 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName b 2024-12-02T22:24:21,476 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:21,476 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:21,476 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,477 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName c 2024-12-02T22:24:21,477 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:21,478 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:21,478 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,478 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,479 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,479 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,480 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:21,481 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,485 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:24:21,485 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 39c3fea49fafaa066e452b3008974b5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62360631, jitterRate=-0.07075418531894684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:21,485 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 39c3fea49fafaa066e452b3008974b5a: Writing region info on filesystem at 1733178261470Initializing all the Stores at 1733178261471 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178261471Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178261472 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178261472Cleaning up temporary data from old regions at 1733178261479 (+7 ms)Region opened successfully at 1733178261485 (+6 ms) 2024-12-02T22:24:21,485 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 39c3fea49fafaa066e452b3008974b5a, disabling compactions & flushes 2024-12-02T22:24:21,485 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,486 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,486 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. after waiting 0 ms 2024-12-02T22:24:21,486 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,489 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,489 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 39c3fea49fafaa066e452b3008974b5a: Waiting for close lock at 1733178261485Disabling compacts and flushes for region at 1733178261485Disabling writes for close at 1733178261486 (+1 ms)Writing region close event to WAL at 1733178261489 (+3 ms)Closed at 1733178261489 2024-12-02T22:24:21,499 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-31902826/hregion-31902826.1733178260813 not finished, retry = 0 2024-12-02T22:24:21,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741930_1109 (size=175) 2024-12-02T22:24:21,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741930_1109 (size=175) 2024-12-02T22:24:21,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741930_1109 (size=175) 2024-12-02T22:24:21,602 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:24:21,602 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-31902826:(num 1733178260813) 2024-12-02T22:24:21,602 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:21,604 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:21,626 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605, exclude list is [], retry=0 2024-12-02T22:24:21,628 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:21,629 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:21,629 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:21,657 DEBUG [master/7ae73638d47e:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-02T22:24:21,658 DEBUG [master/7ae73638d47e:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5f517d31aff1d4768327a1ec4cf716a7 changed from -1.0 to 0.0, refreshing cache 2024-12-02T22:24:21,675 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605 2024-12-02T22:24:21,676 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:24:21,676 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 39c3fea49fafaa066e452b3008974b5a, NAME => 'testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:21,676 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:21,676 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,676 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,680 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,681 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName a 2024-12-02T22:24:21,681 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:21,682 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:21,682 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,683 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName b 2024-12-02T22:24:21,683 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:21,683 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:21,684 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,684 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName c 2024-12-02T22:24:21,684 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:21,685 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:21,685 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,686 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,687 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,689 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,689 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,690 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:21,691 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:21,693 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 39c3fea49fafaa066e452b3008974b5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58890814, jitterRate=-0.12245848774909973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:21,693 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 39c3fea49fafaa066e452b3008974b5a: Writing region info on filesystem at 1733178261676Initializing all the Stores at 1733178261677 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178261677Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178261680 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178261680Cleaning up temporary data from old regions at 1733178261689 (+9 ms)Region opened successfully at 1733178261693 (+4 ms) 2024-12-02T22:24:21,704 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39c3fea49fafaa066e452b3008974b5a 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-02T22:24:21,728 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/a/abcd10129a304504890afef970cacc1d is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733178261693/Put/seqid=0 2024-12-02T22:24:21,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741933_1112 (size=5958) 2024-12-02T22:24:21,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741933_1112 (size=5958) 2024-12-02T22:24:21,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741933_1112 (size=5958) 2024-12-02T22:24:21,745 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/a/abcd10129a304504890afef970cacc1d 2024-12-02T22:24:21,761 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/a/abcd10129a304504890afef970cacc1d as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/a/abcd10129a304504890afef970cacc1d 2024-12-02T22:24:21,768 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/a/abcd10129a304504890afef970cacc1d, entries=10, sequenceid=13, filesize=5.8 K 2024-12-02T22:24:21,770 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 39c3fea49fafaa066e452b3008974b5a in 65ms, sequenceid=13, compaction requested=false 2024-12-02T22:24:21,770 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 39c3fea49fafaa066e452b3008974b5a: 2024-12-02T22:24:21,800 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 39c3fea49fafaa066e452b3008974b5a, disabling compactions & flushes 2024-12-02T22:24:21,800 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,800 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,800 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. after waiting 0 ms 2024-12-02T22:24:21,801 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,809 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,809 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:21,809 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 39c3fea49fafaa066e452b3008974b5a: Waiting for close lock at 1733178261800Disabling compacts and flushes for region at 1733178261800Disabling writes for close at 1733178261800Writing region close event to WAL at 1733178261809 (+9 ms)Closed at 1733178261809 2024-12-02T22:24:21,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741932_1111 (size=6243) 2024-12-02T22:24:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741932_1111 (size=6243) 2024-12-02T22:24:21,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741932_1111 (size=6243) 2024-12-02T22:24:21,855 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605, size=6.1 K (6243bytes) 2024-12-02T22:24:21,855 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605 2024-12-02T22:24:21,856 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605 after 1ms 2024-12-02T22:24:21,901 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:21,901 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605 took 46ms 2024-12-02T22:24:21,904 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605 so closing down 2024-12-02T22:24:21,904 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:24:21,909 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733178261605.temp 2024-12-02T22:24:21,956 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000003-wal.1733178261605.temp 2024-12-02T22:24:21,957 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:24:21,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741934_1113 (size=5575) 2024-12-02T22:24:21,971 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000003-wal.1733178261605.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-02T22:24:21,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741934_1113 (size=5575) 2024-12-02T22:24:21,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741934_1113 (size=5575) 2024-12-02T22:24:21,973 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000003-wal.1733178261605.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000035 2024-12-02T22:24:21,973 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 71 ms; skipped=2; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-02T22:24:21,973 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605, journal: Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605, size=6.1 K (6243bytes) at 1733178261855Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605 so closing down at 1733178261904 (+49 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000003-wal.1733178261605.temp at 1733178261956 (+52 ms)3 split writer threads finished at 1733178261957 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000003-wal.1733178261605.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733178261971 (+14 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000003-wal.1733178261605.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000035 at 1733178261973 (+2 ms)Processed 32 edits across 1 Regions in 71 ms; skipped=2; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733178261973 2024-12-02T22:24:21,981 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261605 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178261605 2024-12-02T22:24:21,982 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000035 2024-12-02T22:24:21,982 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:21,984 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:22,005 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984, exclude list is [], retry=0 2024-12-02T22:24:22,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:22,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:22,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:22,057 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 2024-12-02T22:24:22,058 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917),(127.0.0.1/127.0.0.1:45007:45007)] 2024-12-02T22:24:22,058 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 39c3fea49fafaa066e452b3008974b5a, NAME => 'testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:24:22,058 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:22,059 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,059 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,060 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,061 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName a 2024-12-02T22:24:22,061 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:22,082 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/a/abcd10129a304504890afef970cacc1d 2024-12-02T22:24:22,083 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:22,083 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,084 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName b 2024-12-02T22:24:22,084 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:22,084 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:22,084 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,085 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName c 2024-12-02T22:24:22,085 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:22,086 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:22,086 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,087 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,088 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,089 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000035 2024-12-02T22:24:22,132 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:22,133 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000035 2024-12-02T22:24:22,134 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39c3fea49fafaa066e452b3008974b5a 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-02T22:24:22,159 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/b/63eb825bb79843d085ff28b287891858 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733178261770/Put/seqid=0 2024-12-02T22:24:22,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741936_1115 (size=5958) 2024-12-02T22:24:22,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741936_1115 (size=5958) 2024-12-02T22:24:22,212 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/b/63eb825bb79843d085ff28b287891858 2024-12-02T22:24:22,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741936_1115 (size=5958) 2024-12-02T22:24:22,255 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/c/84e40d0d18f5430b92e9bb272bd45508 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733178261779/Put/seqid=0 2024-12-02T22:24:22,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741937_1116 (size=5958) 2024-12-02T22:24:22,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741937_1116 (size=5958) 2024-12-02T22:24:22,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741937_1116 (size=5958) 2024-12-02T22:24:22,287 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/c/84e40d0d18f5430b92e9bb272bd45508 2024-12-02T22:24:22,295 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/b/63eb825bb79843d085ff28b287891858 as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/b/63eb825bb79843d085ff28b287891858 2024-12-02T22:24:22,301 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/b/63eb825bb79843d085ff28b287891858, entries=10, sequenceid=35, filesize=5.8 K 2024-12-02T22:24:22,303 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/c/84e40d0d18f5430b92e9bb272bd45508 as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/c/84e40d0d18f5430b92e9bb272bd45508 2024-12-02T22:24:22,310 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/c/84e40d0d18f5430b92e9bb272bd45508, entries=10, sequenceid=35, filesize=5.8 K 2024-12-02T22:24:22,311 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 39c3fea49fafaa066e452b3008974b5a in 178ms, sequenceid=35, compaction requested=false; wal=null 2024-12-02T22:24:22,312 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000035 2024-12-02T22:24:22,313 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,313 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,314 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:22,315 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:22,321 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-02T22:24:22,322 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 39c3fea49fafaa066e452b3008974b5a; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68438154, jitterRate=0.01980796456336975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:22,323 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 39c3fea49fafaa066e452b3008974b5a: Writing region info on filesystem at 1733178262059Initializing all the Stores at 1733178262060 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178262060Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178262060Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178262060Obtaining lock to block concurrent updates at 1733178262134 (+74 ms)Preparing flush snapshotting stores in 39c3fea49fafaa066e452b3008974b5a at 1733178262134Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733178262134Flushing stores of testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. at 1733178262134Flushing 39c3fea49fafaa066e452b3008974b5a/b: creating writer at 1733178262134Flushing 39c3fea49fafaa066e452b3008974b5a/b: appending metadata at 1733178262158 (+24 ms)Flushing 39c3fea49fafaa066e452b3008974b5a/b: closing flushed file at 1733178262158Flushing 39c3fea49fafaa066e452b3008974b5a/c: creating writer at 1733178262220 (+62 ms)Flushing 39c3fea49fafaa066e452b3008974b5a/c: appending metadata at 1733178262254 (+34 ms)Flushing 39c3fea49fafaa066e452b3008974b5a/c: closing flushed file at 1733178262254Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28a42d43: reopening flushed file at 1733178262294 (+40 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@534a4e0c: reopening flushed file at 1733178262302 (+8 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 39c3fea49fafaa066e452b3008974b5a in 178ms, sequenceid=35, compaction requested=false; wal=null at 1733178262311 (+9 ms)Cleaning up temporary data from old regions at 1733178262313 (+2 ms)Region opened successfully at 1733178262322 (+9 ms) 2024-12-02T22:24:22,446 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984, size=0 (0bytes) 2024-12-02T22:24:22,446 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 might be still open, length is 0 2024-12-02T22:24:22,446 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 2024-12-02T22:24:22,447 WARN [IPC Server handler 4 on default port 38075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-02T22:24:22,448 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 after 2ms 2024-12-02T22:24:23,196 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:43198 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:40725:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43198 dst: /127.0.0.1:40725 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40725 remote=/127.0.0.1:43198]. Total timeout mills is 60000, 59174 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:23,197 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:47356 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37315:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47356 dst: /127.0.0.1:37315 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:23,197 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-545031594_22 at /127.0.0.1:47070 [Receiving block BP-981501226-172.17.0.2-1733178187608:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34817:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47070 dst: /127.0.0.1:34817 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:23,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741935_1117 (size=5567) 2024-12-02T22:24:23,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741935_1117 (size=5567) 2024-12-02T22:24:26,449 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 after 4003ms 2024-12-02T22:24:26,518 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:26,518 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 took 4072ms 2024-12-02T22:24:26,525 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984; continuing. 2024-12-02T22:24:26,525 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 so closing down 2024-12-02T22:24:26,525 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T22:24:26,535 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733178261984.temp 2024-12-02T22:24:26,578 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000037-wal.1733178261984.temp 2024-12-02T22:24:26,579 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T22:24:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741938_1118 (size=5575) 2024-12-02T22:24:26,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741938_1118 (size=5575) 2024-12-02T22:24:26,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741938_1118 (size=5575) 2024-12-02T22:24:26,606 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000037-wal.1733178261984.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-02T22:24:26,608 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000037-wal.1733178261984.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000066 2024-12-02T22:24:26,608 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 87 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T22:24:26,608 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984, journal: Splitting hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984, size=0 (0bytes) at 1733178262446Finishing writing output for hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 so closing down at 1733178266525 (+4079 ms)Creating recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000037-wal.1733178261984.temp at 1733178266578 (+53 ms)3 split writer threads finished at 1733178266579 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000037-wal.1733178261984.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733178266606 (+27 ms)Rename recovered edits hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000037-wal.1733178261984.temp to hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000066 at 1733178266608 (+2 ms)Processed 30 edits across 1 Regions in 87 ms; skipped=0; WAL=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984, size=0, length=0, corrupted=false, cancelled=false at 1733178266608 2024-12-02T22:24:26,611 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 to hdfs://localhost:38075/hbase/oldWALs/wal.1733178261984 2024-12-02T22:24:26,612 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000066 2024-12-02T22:24:26,612 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T22:24:26,614 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38075/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805, archiveDir=hdfs://localhost:38075/hbase/oldWALs, maxLogs=32 2024-12-02T22:24:26,641 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178266614, exclude list is [], retry=0 2024-12-02T22:24:26,644 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34817,DS-c6cfc7c4-7749-4bb4-b739-c25fa5da246c,DISK] 2024-12-02T22:24:26,645 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40725,DS-4c921f6b-bd48-4e8d-9fe7-1318810fdfdd,DISK] 2024-12-02T22:24:26,645 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37315,DS-57e02b84-9b94-4395-a6ba-37fe566993d5,DISK] 2024-12-02T22:24:26,702 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178266614 2024-12-02T22:24:26,703 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45007:45007),(127.0.0.1/127.0.0.1:36155:36155),(127.0.0.1/127.0.0.1:37917:37917)] 2024-12-02T22:24:26,704 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:24:26,707 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,708 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName a 2024-12-02T22:24:26,708 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:26,735 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/a/abcd10129a304504890afef970cacc1d 2024-12-02T22:24:26,735 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:26,736 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,737 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName b 2024-12-02T22:24:26,737 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:26,744 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/b/63eb825bb79843d085ff28b287891858 2024-12-02T22:24:26,744 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:26,744 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,750 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c3fea49fafaa066e452b3008974b5a columnFamilyName c 2024-12-02T22:24:26,750 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:24:26,791 DEBUG [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/c/84e40d0d18f5430b92e9bb272bd45508 2024-12-02T22:24:26,791 INFO [StoreOpener-39c3fea49fafaa066e452b3008974b5a-1 {}] regionserver.HStore(327): Store=39c3fea49fafaa066e452b3008974b5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:24:26,796 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,797 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,800 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000066 2024-12-02T22:24:26,842 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T22:24:26,849 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000066 2024-12-02T22:24:26,849 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39c3fea49fafaa066e452b3008974b5a 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-02T22:24:26,864 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/a/905c4901bbc045a89fe58a4214edc22c is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733178262331/Put/seqid=0 2024-12-02T22:24:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741940_1120 (size=5958) 2024-12-02T22:24:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741940_1120 (size=5958) 2024-12-02T22:24:26,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741940_1120 (size=5958) 2024-12-02T22:24:26,871 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/a/905c4901bbc045a89fe58a4214edc22c 2024-12-02T22:24:26,897 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/b/ef9fc6405d86451592d7e60a798a2030 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733178262343/Put/seqid=0 2024-12-02T22:24:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741941_1121 (size=5958) 2024-12-02T22:24:26,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741941_1121 (size=5958) 2024-12-02T22:24:26,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741941_1121 (size=5958) 2024-12-02T22:24:26,905 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/b/ef9fc6405d86451592d7e60a798a2030 2024-12-02T22:24:26,929 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/c/642fa516dee640ed97421224489fde42 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733178262355/Put/seqid=0 2024-12-02T22:24:26,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741942_1122 (size=5958) 2024-12-02T22:24:26,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741942_1122 (size=5958) 2024-12-02T22:24:26,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741942_1122 (size=5958) 2024-12-02T22:24:26,938 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/c/642fa516dee640ed97421224489fde42 2024-12-02T22:24:26,943 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/a/905c4901bbc045a89fe58a4214edc22c as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/a/905c4901bbc045a89fe58a4214edc22c 2024-12-02T22:24:26,948 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/a/905c4901bbc045a89fe58a4214edc22c, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T22:24:26,953 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/b/ef9fc6405d86451592d7e60a798a2030 as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/b/ef9fc6405d86451592d7e60a798a2030 2024-12-02T22:24:26,958 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/b/ef9fc6405d86451592d7e60a798a2030, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T22:24:26,959 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/.tmp/c/642fa516dee640ed97421224489fde42 as hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/c/642fa516dee640ed97421224489fde42 2024-12-02T22:24:26,971 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/c/642fa516dee640ed97421224489fde42, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T22:24:26,972 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 39c3fea49fafaa066e452b3008974b5a in 123ms, sequenceid=66, compaction requested=false; wal=null 2024-12-02T22:24:26,972 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/0000000000000000066 2024-12-02T22:24:26,974 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,974 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,974 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T22:24:26,976 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 39c3fea49fafaa066e452b3008974b5a 2024-12-02T22:24:26,978 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/hbase/data/default/testReplayEditsWrittenViaHRegion/39c3fea49fafaa066e452b3008974b5a/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-02T22:24:26,979 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 39c3fea49fafaa066e452b3008974b5a; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59922315, jitterRate=-0.10708792507648468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T22:24:26,979 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 39c3fea49fafaa066e452b3008974b5a: Writing region info on filesystem at 1733178266704Initializing all the Stores at 1733178266705 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178266705Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178266706 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733178266706Obtaining lock to block concurrent updates at 1733178266849 (+143 ms)Preparing flush snapshotting stores in 39c3fea49fafaa066e452b3008974b5a at 1733178266849Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733178266849Flushing stores of testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. at 1733178266849Flushing 39c3fea49fafaa066e452b3008974b5a/a: creating writer at 1733178266849Flushing 39c3fea49fafaa066e452b3008974b5a/a: appending metadata at 1733178266863 (+14 ms)Flushing 39c3fea49fafaa066e452b3008974b5a/a: closing flushed file at 1733178266863Flushing 39c3fea49fafaa066e452b3008974b5a/b: creating writer at 1733178266876 (+13 ms)Flushing 39c3fea49fafaa066e452b3008974b5a/b: appending metadata at 1733178266897 (+21 ms)Flushing 39c3fea49fafaa066e452b3008974b5a/b: closing flushed file at 1733178266897Flushing 39c3fea49fafaa066e452b3008974b5a/c: creating writer at 1733178266911 (+14 ms)Flushing 39c3fea49fafaa066e452b3008974b5a/c: appending metadata at 1733178266928 (+17 ms)Flushing 39c3fea49fafaa066e452b3008974b5a/c: closing flushed file at 1733178266928Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4957476e: reopening flushed file at 1733178266943 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4df163b3: reopening flushed file at 1733178266949 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@441a387a: reopening flushed file at 1733178266958 (+9 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 39c3fea49fafaa066e452b3008974b5a in 123ms, sequenceid=66, compaction requested=false; wal=null at 1733178266972 (+14 ms)Cleaning up temporary data from old regions at 1733178266974 (+2 ms)Region opened successfully at 1733178266979 (+5 ms) 2024-12-02T22:24:26,992 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 39c3fea49fafaa066e452b3008974b5a, disabling compactions & flushes 2024-12-02T22:24:26,992 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:26,992 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:26,992 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. after waiting 0 ms 2024-12-02T22:24:26,992 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:26,994 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733178260806.39c3fea49fafaa066e452b3008974b5a. 2024-12-02T22:24:26,994 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 39c3fea49fafaa066e452b3008974b5a: Waiting for close lock at 1733178266992Disabling compacts and flushes for region at 1733178266992Disabling writes for close at 1733178266992Writing region close event to WAL at 1733178266994 (+2 ms)Closed at 1733178266994 2024-12-02T22:24:26,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741939_1119 (size=175) 2024-12-02T22:24:26,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741939_1119 (size=175) 2024-12-02T22:24:26,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741939_1119 (size=175) 2024-12-02T22:24:27,000 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T22:24:27,000 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733178266614) 2024-12-02T22:24:27,017 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=430 (was 422) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_762949774_22 at /127.0.0.1:43226 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_762949774_22 at /127.0.0.1:47404 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:38075 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_762949774_22 at /127.0.0.1:47122 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:38075 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1402 (was 1340) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=922 (was 933), ProcessCount=11 (was 11), AvailableMemoryMB=3755 (was 3943) 2024-12-02T22:24:27,018 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1402 is superior to 1024 2024-12-02T22:24:27,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T22:24:27,018 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:24:27,018 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:24:27,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:24:27,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:24:27,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:24:27,019 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T22:24:27,019 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1811970755, stopped=false 2024-12-02T22:24:27,020 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7ae73638d47e,42475,1733178195164 2024-12-02T22:24:27,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T22:24:27,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T22:24:27,022 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T22:24:27,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:24:27,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:24:27,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T22:24:27,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:24:27,024 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:24:27,024 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:24:27,024 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:24:27,024 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:24:27,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:24:27,024 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7ae73638d47e,35511,1733178196345' ***** 2024-12-02T22:24:27,024 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T22:24:27,024 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7ae73638d47e,45899,1733178196419' ***** 2024-12-02T22:24:27,024 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T22:24:27,025 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T22:24:27,025 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T22:24:27,025 INFO [RS:2;7ae73638d47e:45899 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T22:24:27,025 INFO [RS:2;7ae73638d47e:45899 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T22:24:27,025 INFO [RS:1;7ae73638d47e:35511 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T22:24:27,025 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(959): stopping server 7ae73638d47e,45899,1733178196419 2024-12-02T22:24:27,025 INFO [RS:1;7ae73638d47e:35511 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T22:24:27,025 INFO [RS:2;7ae73638d47e:45899 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T22:24:27,025 INFO [RS:2;7ae73638d47e:45899 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7ae73638d47e:45899. 2024-12-02T22:24:27,025 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:24:27,025 DEBUG [RS:2;7ae73638d47e:45899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:24:27,025 DEBUG [RS:2;7ae73638d47e:45899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:24:27,025 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(959): stopping server 7ae73638d47e,35511,1733178196345 2024-12-02T22:24:27,025 INFO [RS:1;7ae73638d47e:35511 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T22:24:27,025 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(976): stopping server 7ae73638d47e,45899,1733178196419; all regions closed. 2024-12-02T22:24:27,026 INFO [RS:1;7ae73638d47e:35511 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7ae73638d47e:35511. 2024-12-02T22:24:27,026 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f517d31aff1d4768327a1ec4cf716a7, disabling compactions & flushes 2024-12-02T22:24:27,026 DEBUG [RS:1;7ae73638d47e:35511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:24:27,026 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:24:27,026 DEBUG [RS:1;7ae73638d47e:35511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:24:27,026 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:24:27,026 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. after waiting 0 ms 2024-12-02T22:24:27,026 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T22:24:27,026 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:24:27,026 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T22:24:27,026 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T22:24:27,026 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T22:24:27,026 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T22:24:27,026 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T22:24:27,027 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:24:27,027 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T22:24:27,027 DEBUG [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5f517d31aff1d4768327a1ec4cf716a7=testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7.} 2024-12-02T22:24:27,027 DEBUG [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5f517d31aff1d4768327a1ec4cf716a7 2024-12-02T22:24:27,028 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T22:24:27,028 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T22:24:27,028 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T22:24:27,028 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T22:24:27,028 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T22:24:27,028 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-02T22:24:27,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741833_1009 (size=175) 2024-12-02T22:24:27,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741833_1009 (size=175) 2024-12-02T22:24:27,037 DEBUG [RS:2;7ae73638d47e:45899 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7ae73638d47e%2C45899%2C1733178196419:(num 1733178200028) 2024-12-02T22:24:27,038 DEBUG [RS:2;7ae73638d47e:45899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] hbase.ChoreService(370): Chore service for: regionserver/7ae73638d47e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T22:24:27,038 INFO [RS:2;7ae73638d47e:45899 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45899 2024-12-02T22:24:27,039 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/info/0b8b3d2930004247a669c87636dc5f9d is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7./info:regioninfo/1733178236879/Put/seqid=0 2024-12-02T22:24:27,039 INFO [regionserver/7ae73638d47e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T22:24:27,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7ae73638d47e,45899,1733178196419 2024-12-02T22:24:27,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T22:24:27,043 INFO [RS:2;7ae73638d47e:45899 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T22:24:27,048 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5f517d31aff1d4768327a1ec4cf716a7/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-02T22:24:27,049 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7ae73638d47e,45899,1733178196419] 2024-12-02T22:24:27,050 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:24:27,050 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f517d31aff1d4768327a1ec4cf716a7: Waiting for close lock at 1733178267025Running coprocessor pre-close hooks at 1733178267025Disabling compacts and flushes for region at 1733178267025Disabling writes for close at 1733178267026 (+1 ms)Writing region close event to WAL at 1733178267030 (+4 ms)Running coprocessor post-close hooks at 1733178267049 (+19 ms)Closed at 1733178267050 (+1 ms) 2024-12-02T22:24:27,050 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733178229821.5f517d31aff1d4768327a1ec4cf716a7. 2024-12-02T22:24:27,051 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7ae73638d47e,45899,1733178196419 already deleted, retry=false 2024-12-02T22:24:27,051 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7ae73638d47e,45899,1733178196419 expired; onlineServers=1 2024-12-02T22:24:27,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741943_1123 (size=6778) 2024-12-02T22:24:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741943_1123 (size=6778) 2024-12-02T22:24:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741943_1123 (size=6778) 2024-12-02T22:24:27,058 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/info/0b8b3d2930004247a669c87636dc5f9d 2024-12-02T22:24:27,065 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/.tmp/info/0b8b3d2930004247a669c87636dc5f9d as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/info/0b8b3d2930004247a669c87636dc5f9d 2024-12-02T22:24:27,078 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/info/0b8b3d2930004247a669c87636dc5f9d, entries=8, sequenceid=23, filesize=6.6 K 2024-12-02T22:24:27,081 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 53ms, sequenceid=23, compaction requested=false 2024-12-02T22:24:27,092 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-02T22:24:27,092 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T22:24:27,092 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T22:24:27,092 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733178267028Running coprocessor pre-close hooks at 1733178267028Disabling compacts and flushes for region at 1733178267028Disabling writes for close at 1733178267028Obtaining lock to block concurrent updates at 1733178267028Preparing flush snapshotting stores in 1588230740 at 1733178267028Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733178267029 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733178267029Flushing 1588230740/info: creating writer at 1733178267030 (+1 ms)Flushing 1588230740/info: appending metadata at 1733178267038 (+8 ms)Flushing 1588230740/info: closing flushed file at 1733178267038Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1338dc05: reopening flushed file at 1733178267065 (+27 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 53ms, sequenceid=23, compaction requested=false at 1733178267081 (+16 ms)Writing region close event to WAL at 1733178267086 (+5 ms)Running coprocessor post-close hooks at 1733178267092 (+6 ms)Closed at 1733178267092 2024-12-02T22:24:27,093 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T22:24:27,096 INFO [regionserver/7ae73638d47e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T22:24:27,097 INFO [regionserver/7ae73638d47e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T22:24:27,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:24:27,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45899-0x1009d16841a0003, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:24:27,151 INFO [RS:2;7ae73638d47e:45899 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T22:24:27,151 INFO [RS:2;7ae73638d47e:45899 {}] regionserver.HRegionServer(1031): Exiting; stopping=7ae73638d47e,45899,1733178196419; zookeeper connection closed. 2024-12-02T22:24:27,152 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@190f471b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@190f471b 2024-12-02T22:24:27,228 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(976): stopping server 7ae73638d47e,35511,1733178196345; all regions closed. 2024-12-02T22:24:27,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741893_1071 (size=2625) 2024-12-02T22:24:27,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741893_1071 (size=2625) 2024-12-02T22:24:27,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741893_1071 (size=2625) 2024-12-02T22:24:27,234 DEBUG [RS:1;7ae73638d47e:35511 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs 2024-12-02T22:24:27,234 INFO [RS:1;7ae73638d47e:35511 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7ae73638d47e%2C35511%2C1733178196345.meta:.meta(num 1733178235747) 2024-12-02T22:24:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741835_1011 (size=3621) 2024-12-02T22:24:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741835_1011 (size=3621) 2024-12-02T22:24:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741835_1011 (size=3621) 2024-12-02T22:24:27,240 DEBUG [RS:1;7ae73638d47e:35511 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/oldWALs 2024-12-02T22:24:27,240 INFO [RS:1;7ae73638d47e:35511 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7ae73638d47e%2C35511%2C1733178196345:(num 1733178200029) 2024-12-02T22:24:27,240 DEBUG [RS:1;7ae73638d47e:35511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:24:27,240 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T22:24:27,240 INFO [RS:1;7ae73638d47e:35511 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T22:24:27,241 INFO [RS:1;7ae73638d47e:35511 {}] hbase.ChoreService(370): Chore service for: regionserver/7ae73638d47e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-02T22:24:27,241 INFO [RS:1;7ae73638d47e:35511 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T22:24:27,241 INFO [regionserver/7ae73638d47e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T22:24:27,241 INFO [RS:1;7ae73638d47e:35511 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35511 2024-12-02T22:24:27,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T22:24:27,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7ae73638d47e,35511,1733178196345 2024-12-02T22:24:27,247 INFO [RS:1;7ae73638d47e:35511 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T22:24:27,249 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7ae73638d47e,35511,1733178196345] 2024-12-02T22:24:27,250 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7ae73638d47e,35511,1733178196345 already deleted, retry=false 2024-12-02T22:24:27,250 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7ae73638d47e,35511,1733178196345 expired; onlineServers=0 2024-12-02T22:24:27,251 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7ae73638d47e,42475,1733178195164' ***** 2024-12-02T22:24:27,251 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T22:24:27,251 INFO [M:0;7ae73638d47e:42475 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T22:24:27,251 INFO [M:0;7ae73638d47e:42475 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T22:24:27,251 DEBUG [M:0;7ae73638d47e:42475 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T22:24:27,251 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T22:24:27,251 DEBUG [M:0;7ae73638d47e:42475 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T22:24:27,251 DEBUG [master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.small.0-1733178199086 {}] cleaner.HFileCleaner(306): Exit Thread[master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.small.0-1733178199086,5,FailOnTimeoutGroup] 2024-12-02T22:24:27,251 DEBUG [master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.large.0-1733178199063 {}] cleaner.HFileCleaner(306): Exit Thread[master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.large.0-1733178199063,5,FailOnTimeoutGroup] 2024-12-02T22:24:27,252 INFO [M:0;7ae73638d47e:42475 {}] hbase.ChoreService(370): Chore service for: master/7ae73638d47e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T22:24:27,252 INFO [M:0;7ae73638d47e:42475 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T22:24:27,252 DEBUG [M:0;7ae73638d47e:42475 {}] master.HMaster(1795): Stopping service threads 2024-12-02T22:24:27,252 INFO [M:0;7ae73638d47e:42475 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T22:24:27,252 INFO [M:0;7ae73638d47e:42475 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T22:24:27,253 INFO [M:0;7ae73638d47e:42475 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T22:24:27,253 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T22:24:27,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T22:24:27,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:24:27,259 DEBUG [M:0;7ae73638d47e:42475 {}] zookeeper.ZKUtil(347): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T22:24:27,259 WARN [M:0;7ae73638d47e:42475 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T22:24:27,259 INFO [M:0;7ae73638d47e:42475 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/.lastflushedseqids 2024-12-02T22:24:27,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741944_1124 (size=119) 2024-12-02T22:24:27,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741944_1124 (size=119) 2024-12-02T22:24:27,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741944_1124 (size=119) 2024-12-02T22:24:27,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:24:27,349 INFO [RS:1;7ae73638d47e:35511 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T22:24:27,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35511-0x1009d16841a0002, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:24:27,349 INFO [RS:1;7ae73638d47e:35511 {}] regionserver.HRegionServer(1031): Exiting; stopping=7ae73638d47e,35511,1733178196345; zookeeper connection closed. 2024-12-02T22:24:27,349 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@25b4a096 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@25b4a096 2024-12-02T22:24:27,350 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-02T22:24:27,681 INFO [M:0;7ae73638d47e:42475 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T22:24:27,681 INFO [M:0;7ae73638d47e:42475 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T22:24:27,681 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T22:24:27,681 INFO [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:24:27,681 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:24:27,681 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T22:24:27,681 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:24:27,682 INFO [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-12-02T22:24:27,698 DEBUG [M:0;7ae73638d47e:42475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3a595c6d6bd2454ba469c57916c7bb3b is 82, key is hbase:meta,,1/info:regioninfo/1733178236128/Put/seqid=0 2024-12-02T22:24:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741945_1125 (size=6063) 2024-12-02T22:24:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741945_1125 (size=6063) 2024-12-02T22:24:27,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741945_1125 (size=6063) 2024-12-02T22:24:27,716 INFO [M:0;7ae73638d47e:42475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3a595c6d6bd2454ba469c57916c7bb3b 2024-12-02T22:24:27,738 DEBUG [M:0;7ae73638d47e:42475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5aba18c46c6b480e86085834c567ca86 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733178230777/Put/seqid=0 2024-12-02T22:24:27,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741946_1126 (size=7906) 2024-12-02T22:24:27,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741946_1126 (size=7906) 2024-12-02T22:24:27,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741946_1126 (size=7906) 2024-12-02T22:24:27,760 INFO [M:0;7ae73638d47e:42475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5aba18c46c6b480e86085834c567ca86 2024-12-02T22:24:27,764 INFO [M:0;7ae73638d47e:42475 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5aba18c46c6b480e86085834c567ca86 2024-12-02T22:24:27,781 DEBUG [M:0;7ae73638d47e:42475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/583b7f9200544aeaa337c30af425f0e2 is 69, key is 7ae73638d47e,35511,1733178196345/rs:state/1733178199198/Put/seqid=0 2024-12-02T22:24:27,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741947_1127 (size=5445) 2024-12-02T22:24:27,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741947_1127 (size=5445) 2024-12-02T22:24:27,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741947_1127 (size=5445) 2024-12-02T22:24:27,792 INFO [M:0;7ae73638d47e:42475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/583b7f9200544aeaa337c30af425f0e2 2024-12-02T22:24:27,797 INFO [M:0;7ae73638d47e:42475 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 583b7f9200544aeaa337c30af425f0e2 2024-12-02T22:24:27,798 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3a595c6d6bd2454ba469c57916c7bb3b as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3a595c6d6bd2454ba469c57916c7bb3b 2024-12-02T22:24:27,812 INFO [M:0;7ae73638d47e:42475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3a595c6d6bd2454ba469c57916c7bb3b, entries=14, sequenceid=207, filesize=5.9 K 2024-12-02T22:24:27,817 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5aba18c46c6b480e86085834c567ca86 as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5aba18c46c6b480e86085834c567ca86 2024-12-02T22:24:27,823 INFO [M:0;7ae73638d47e:42475 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5aba18c46c6b480e86085834c567ca86 2024-12-02T22:24:27,824 INFO [M:0;7ae73638d47e:42475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5aba18c46c6b480e86085834c567ca86, entries=21, sequenceid=207, filesize=7.7 K 2024-12-02T22:24:27,825 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/583b7f9200544aeaa337c30af425f0e2 as hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/583b7f9200544aeaa337c30af425f0e2 2024-12-02T22:24:27,840 INFO [M:0;7ae73638d47e:42475 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 583b7f9200544aeaa337c30af425f0e2 2024-12-02T22:24:27,840 INFO [M:0;7ae73638d47e:42475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38075/user/jenkins/test-data/bf96f92f-a2a2-4145-7c3e-f57f66cef8d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/583b7f9200544aeaa337c30af425f0e2, entries=3, sequenceid=207, filesize=5.3 K 2024-12-02T22:24:27,844 INFO [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 163ms, sequenceid=207, compaction requested=false 2024-12-02T22:24:27,845 INFO [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:24:27,845 DEBUG [M:0;7ae73638d47e:42475 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733178267681Disabling compacts and flushes for region at 1733178267681Disabling writes for close at 1733178267681Obtaining lock to block concurrent updates at 1733178267682 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733178267682Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1733178267682Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733178267683 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733178267683Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733178267698 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733178267698Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733178267723 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733178267738 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733178267738Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733178267764 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733178267781 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733178267781Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@668ba8f1: reopening flushed file at 1733178267797 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bcdc63e: reopening flushed file at 1733178267812 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61384da0: reopening flushed file at 1733178267824 (+12 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 163ms, sequenceid=207, compaction requested=false at 1733178267844 (+20 ms)Writing region close event to WAL at 1733178267845 (+1 ms)Closed at 1733178267845 2024-12-02T22:24:27,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40725 is added to blk_1073741830_1006 (size=102162) 2024-12-02T22:24:27,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37315 is added to blk_1073741830_1006 (size=102162) 2024-12-02T22:24:27,860 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T22:24:27,861 INFO [M:0;7ae73638d47e:42475 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T22:24:27,861 INFO [M:0;7ae73638d47e:42475 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42475 2024-12-02T22:24:27,861 INFO [M:0;7ae73638d47e:42475 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T22:24:27,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34817 is added to blk_1073741830_1006 (size=102162) 2024-12-02T22:24:27,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:24:27,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1009d16841a0000, quorum=127.0.0.1:55591, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:24:27,977 INFO [M:0;7ae73638d47e:42475 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T22:24:28,064 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178250653 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178250653 (inode 16688) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733178240143/wal.1733178250653 (inode 16688) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T22:24:28,073 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178239153 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178239153 (inode 16665) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733178237360/wal.1733178239153 (inode 16665) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T22:24:28,074 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733178223037/wal.1733178223630 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:28,081 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178253684 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178253684 (inode 16714) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733178251595/wal.1733178253684 (inode 16714) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T22:24:28,082 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733178254353/wal.1733178254795 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:28,082 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733178260805/wal.1733178261984 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:28,083 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733178207104/wal.1733178207574 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:24:28,085 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal.1733178206411 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal.1733178206411 (inode 16485) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733178204592/wal.1733178206411 (inode 16485) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T22:24:28,090 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal.1733178203348 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal.1733178203348 (inode 16462) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733178201899/wal.1733178203348 (inode 16462) Holder DFSClient_NONMAPREDUCE_-545031594_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T22:24:28,102 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@773b4a40{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T22:24:28,105 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@286fa8e3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T22:24:28,105 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T22:24:28,105 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b0de5e3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T22:24:28,105 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78fad685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir/,STOPPED} 2024-12-02T22:24:28,113 WARN [BP-981501226-172.17.0.2-1733178187608 heartbeating to localhost/127.0.0.1:38075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T22:24:28,113 WARN [BP-981501226-172.17.0.2-1733178187608 heartbeating to localhost/127.0.0.1:38075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-981501226-172.17.0.2-1733178187608 (Datanode Uuid 60203b4e-7dac-4b99-80de-eb0a6d1e92d9) service to localhost/127.0.0.1:38075 2024-12-02T22:24:28,115 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data5/current/BP-981501226-172.17.0.2-1733178187608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T22:24:28,115 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data6/current/BP-981501226-172.17.0.2-1733178187608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T22:24:28,116 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T22:24:28,116 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T22:24:28,117 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T22:24:28,130 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75d96791{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T22:24:28,130 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@60aef58b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T22:24:28,131 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T22:24:28,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@14fd625f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T22:24:28,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78a777d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir/,STOPPED} 2024-12-02T22:24:28,133 WARN [BP-981501226-172.17.0.2-1733178187608 heartbeating to localhost/127.0.0.1:38075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T22:24:28,133 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T22:24:28,133 WARN [BP-981501226-172.17.0.2-1733178187608 heartbeating to localhost/127.0.0.1:38075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-981501226-172.17.0.2-1733178187608 (Datanode Uuid 672d9a78-cfc9-4905-bee0-1d905203b49b) service to localhost/127.0.0.1:38075 2024-12-02T22:24:28,133 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T22:24:28,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data3/current/BP-981501226-172.17.0.2-1733178187608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T22:24:28,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data4/current/BP-981501226-172.17.0.2-1733178187608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T22:24:28,135 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T22:24:28,141 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d480c14{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T22:24:28,141 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8a1e1ee{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T22:24:28,142 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T22:24:28,142 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ace2437{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T22:24:28,142 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49d86915{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir/,STOPPED} 2024-12-02T22:24:28,143 WARN [BP-981501226-172.17.0.2-1733178187608 heartbeating to localhost/127.0.0.1:38075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T22:24:28,143 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T22:24:28,144 WARN [BP-981501226-172.17.0.2-1733178187608 heartbeating to localhost/127.0.0.1:38075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-981501226-172.17.0.2-1733178187608 (Datanode Uuid bce2a7ab-d5be-4824-bd59-025c5634e40b) service to localhost/127.0.0.1:38075 2024-12-02T22:24:28,144 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T22:24:28,144 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data1/current/BP-981501226-172.17.0.2-1733178187608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T22:24:28,144 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/cluster_4c6a19f2-fcd9-8dab-6989-ae4629540274/data/data2/current/BP-981501226-172.17.0.2-1733178187608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T22:24:28,144 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T22:24:28,158 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14f32243{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T22:24:28,158 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71a2a7f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T22:24:28,159 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T22:24:28,159 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@120a4c3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T22:24:28,159 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a6ae9ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cca04233-11ef-93e4-061f-3638a95ff911/hadoop.log.dir/,STOPPED} 2024-12-02T22:24:28,175 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T22:24:28,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down