2024-12-08 01:18:23,810 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-08 01:18:23,822 main DEBUG Took 0.010522 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 01:18:23,823 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 01:18:23,823 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 01:18:23,824 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 01:18:23,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,833 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 01:18:23,846 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,848 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,849 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,849 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,849 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,850 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,851 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,852 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,853 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,854 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,854 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,855 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,856 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,856 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,856 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,857 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,858 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:18:23,859 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,859 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 01:18:23,861 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:18:23,862 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 01:18:23,864 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 01:18:23,864 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 01:18:23,866 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 01:18:23,866 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 01:18:23,875 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 01:18:23,877 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 01:18:23,879 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 01:18:23,879 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 01:18:23,880 main DEBUG createAppenders(={Console}) 2024-12-08 01:18:23,881 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-12-08 01:18:23,881 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-08 01:18:23,882 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-12-08 01:18:23,882 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 01:18:23,883 main DEBUG OutputStream closed 2024-12-08 01:18:23,883 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 01:18:23,883 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 01:18:23,884 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-12-08 01:18:23,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 01:18:23,967 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 01:18:23,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 01:18:23,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 01:18:23,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 01:18:23,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 01:18:23,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 01:18:23,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 01:18:23,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 01:18:23,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 01:18:23,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 01:18:23,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 01:18:23,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 01:18:23,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 01:18:23,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 01:18:23,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 01:18:23,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 01:18:23,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 01:18:23,977 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 01:18:23,978 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-12-08 01:18:23,978 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 01:18:23,979 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-12-08T01:18:24,187 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f 2024-12-08 01:18:24,190 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 01:18:24,191 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T01:18:24,199 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-08T01:18:24,227 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T01:18:24,276 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T01:18:24,276 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T01:18:24,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T01:18:24,300 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8, deleteOnExit=true 2024-12-08T01:18:24,301 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T01:18:24,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/test.cache.data in system properties and HBase conf 2024-12-08T01:18:24,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T01:18:24,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir in system properties and HBase conf 2024-12-08T01:18:24,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T01:18:24,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T01:18:24,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T01:18:24,375 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T01:18:24,452 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T01:18:24,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T01:18:24,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T01:18:24,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T01:18:24,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T01:18:24,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T01:18:24,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T01:18:24,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T01:18:24,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T01:18:24,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T01:18:24,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/nfs.dump.dir in system properties and HBase conf 2024-12-08T01:18:24,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/java.io.tmpdir in system properties and HBase conf 2024-12-08T01:18:24,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T01:18:24,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T01:18:24,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T01:18:25,478 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T01:18:25,541 INFO [Time-limited test {}] log.Log(170): Logging initialized @2338ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T01:18:25,599 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:18:25,650 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:18:25,676 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:18:25,676 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:18:25,677 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T01:18:25,690 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:18:25,692 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:18:25,693 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:18:25,845 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/java.io.tmpdir/jetty-localhost-35117-hadoop-hdfs-3_4_1-tests_jar-_-any-16213498804493232086/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T01:18:25,850 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:35117} 2024-12-08T01:18:25,850 INFO [Time-limited test {}] server.Server(415): Started @2647ms 2024-12-08T01:18:26,359 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:18:26,365 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:18:26,366 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:18:26,366 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:18:26,366 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T01:18:26,367 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:18:26,368 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:18:26,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/java.io.tmpdir/jetty-localhost-35235-hadoop-hdfs-3_4_1-tests_jar-_-any-7813784531160100147/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:18:26,463 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:35235} 2024-12-08T01:18:26,463 INFO [Time-limited test {}] server.Server(415): Started @3260ms 2024-12-08T01:18:26,510 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T01:18:26,615 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:18:26,621 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:18:26,623 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:18:26,623 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:18:26,623 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T01:18:26,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:18:26,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:18:26,731 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/java.io.tmpdir/jetty-localhost-34301-hadoop-hdfs-3_4_1-tests_jar-_-any-1600760357803464024/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:18:26,732 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:34301} 2024-12-08T01:18:26,732 INFO [Time-limited test {}] server.Server(415): Started @3529ms 2024-12-08T01:18:26,734 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T01:18:26,778 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:18:26,785 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:18:26,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:18:26,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:18:26,786 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T01:18:26,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:18:26,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:18:26,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/java.io.tmpdir/jetty-localhost-35897-hadoop-hdfs-3_4_1-tests_jar-_-any-5826946904798973417/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:18:26,880 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:35897} 2024-12-08T01:18:26,880 INFO [Time-limited test {}] server.Server(415): Started @3677ms 2024-12-08T01:18:26,882 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T01:18:28,244 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data4/current/BP-1537938077-172.17.0.2-1733620704924/current, will proceed with Du for space computation calculation, 2024-12-08T01:18:28,244 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data2/current/BP-1537938077-172.17.0.2-1733620704924/current, will proceed with Du for space computation calculation, 2024-12-08T01:18:28,244 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data3/current/BP-1537938077-172.17.0.2-1733620704924/current, will proceed with Du for space computation calculation, 2024-12-08T01:18:28,244 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data1/current/BP-1537938077-172.17.0.2-1733620704924/current, will proceed with Du for space computation calculation, 2024-12-08T01:18:28,276 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T01:18:28,276 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T01:18:28,293 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data5/current/BP-1537938077-172.17.0.2-1733620704924/current, will proceed with Du for space computation calculation, 2024-12-08T01:18:28,293 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data6/current/BP-1537938077-172.17.0.2-1733620704924/current, will proceed with Du for space computation calculation, 2024-12-08T01:18:28,324 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T01:18:28,333 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcff4c2061190f0e3 with lease ID 0xc9914ce05da5ae8a: Processing first storage report for DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e from datanode DatanodeRegistration(127.0.0.1:39423, datanodeUuid=9f7aa09e-f157-41b4-8217-8a9b0a69ef02, infoPort=35469, infoSecurePort=0, ipcPort=43303, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924) 2024-12-08T01:18:28,334 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcff4c2061190f0e3 with lease ID 0xc9914ce05da5ae8a: from storage DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e node DatanodeRegistration(127.0.0.1:39423, datanodeUuid=9f7aa09e-f157-41b4-8217-8a9b0a69ef02, infoPort=35469, infoSecurePort=0, ipcPort=43303, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T01:18:28,334 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc1532ff9eb8bff52 with lease ID 0xc9914ce05da5ae8b: Processing first storage report for DS-ca67abd9-9abf-444b-b8be-d54e57b5b322 from datanode DatanodeRegistration(127.0.0.1:45653, datanodeUuid=d4b0216d-adfc-443e-acf6-d06cad823a5c, infoPort=45421, infoSecurePort=0, ipcPort=34225, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924) 2024-12-08T01:18:28,334 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc1532ff9eb8bff52 with lease ID 0xc9914ce05da5ae8b: from storage DS-ca67abd9-9abf-444b-b8be-d54e57b5b322 node DatanodeRegistration(127.0.0.1:45653, datanodeUuid=d4b0216d-adfc-443e-acf6-d06cad823a5c, infoPort=45421, infoSecurePort=0, ipcPort=34225, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:18:28,334 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x70134f1892cc5b1d with lease ID 0xc9914ce05da5ae8c: Processing first storage report for DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b from datanode DatanodeRegistration(127.0.0.1:46125, datanodeUuid=d35e6347-9819-45b6-b8d9-5ac18385aaf9, infoPort=46499, infoSecurePort=0, ipcPort=43061, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924) 2024-12-08T01:18:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x70134f1892cc5b1d with lease ID 0xc9914ce05da5ae8c: from storage DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b node DatanodeRegistration(127.0.0.1:46125, datanodeUuid=d35e6347-9819-45b6-b8d9-5ac18385aaf9, infoPort=46499, infoSecurePort=0, ipcPort=43061, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:18:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcff4c2061190f0e3 with lease ID 0xc9914ce05da5ae8a: Processing first storage report for DS-3f91bddd-9acd-4abc-bf28-844c3c4e25c3 from datanode DatanodeRegistration(127.0.0.1:39423, datanodeUuid=9f7aa09e-f157-41b4-8217-8a9b0a69ef02, infoPort=35469, infoSecurePort=0, ipcPort=43303, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924) 2024-12-08T01:18:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcff4c2061190f0e3 with lease ID 0xc9914ce05da5ae8a: from storage DS-3f91bddd-9acd-4abc-bf28-844c3c4e25c3 node DatanodeRegistration(127.0.0.1:39423, datanodeUuid=9f7aa09e-f157-41b4-8217-8a9b0a69ef02, infoPort=35469, infoSecurePort=0, ipcPort=43303, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:18:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc1532ff9eb8bff52 with lease ID 0xc9914ce05da5ae8b: Processing first storage report for DS-de1c6a4f-0d35-416f-a337-4a176ea84d78 from datanode DatanodeRegistration(127.0.0.1:45653, datanodeUuid=d4b0216d-adfc-443e-acf6-d06cad823a5c, infoPort=45421, infoSecurePort=0, ipcPort=34225, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924) 2024-12-08T01:18:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc1532ff9eb8bff52 with lease ID 0xc9914ce05da5ae8b: from storage DS-de1c6a4f-0d35-416f-a337-4a176ea84d78 node DatanodeRegistration(127.0.0.1:45653, datanodeUuid=d4b0216d-adfc-443e-acf6-d06cad823a5c, infoPort=45421, infoSecurePort=0, ipcPort=34225, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:18:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x70134f1892cc5b1d with lease ID 0xc9914ce05da5ae8c: Processing first storage report for DS-23df8ff3-01cf-4aac-a299-4e7de243137b from datanode DatanodeRegistration(127.0.0.1:46125, datanodeUuid=d35e6347-9819-45b6-b8d9-5ac18385aaf9, infoPort=46499, infoSecurePort=0, ipcPort=43061, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924) 2024-12-08T01:18:28,336 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x70134f1892cc5b1d with lease ID 0xc9914ce05da5ae8c: from storage DS-23df8ff3-01cf-4aac-a299-4e7de243137b node DatanodeRegistration(127.0.0.1:46125, datanodeUuid=d35e6347-9819-45b6-b8d9-5ac18385aaf9, infoPort=46499, infoSecurePort=0, ipcPort=43061, storageInfo=lv=-57;cid=testClusterID;nsid=174186949;c=1733620704924), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:18:28,397 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f 2024-12-08T01:18:28,467 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/zookeeper_0, clientPort=52656, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T01:18:28,475 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52656 2024-12-08T01:18:28,488 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:28,492 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:28,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741825_1001 (size=7) 2024-12-08T01:18:28,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741825_1001 (size=7) 2024-12-08T01:18:28,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741825_1001 (size=7) 2024-12-08T01:18:29,076 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc with version=8 2024-12-08T01:18:29,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/hbase-staging 2024-12-08T01:18:29,372 INFO [Time-limited test {}] client.ConnectionUtils(128): master/96982154c400:0 server-side Connection retries=45 2024-12-08T01:18:29,380 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:29,380 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:29,384 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:18:29,385 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:29,385 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:18:29,504 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T01:18:29,556 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T01:18:29,564 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T01:18:29,567 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:18:29,589 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 53754 (auto-detected) 2024-12-08T01:18:29,590 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T01:18:29,607 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33573 2024-12-08T01:18:29,625 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33573 connecting to ZooKeeper ensemble=127.0.0.1:52656 2024-12-08T01:18:29,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335730x0, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:18:29,780 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33573-0x1000300bb2c0000 connected 2024-12-08T01:18:29,955 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:29,962 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:29,972 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:18:29,975 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc, hbase.cluster.distributed=false 2024-12-08T01:18:29,996 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:18:30,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33573 2024-12-08T01:18:30,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33573 2024-12-08T01:18:30,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33573 2024-12-08T01:18:30,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33573 2024-12-08T01:18:30,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33573 2024-12-08T01:18:30,088 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/96982154c400:0 server-side Connection retries=45 2024-12-08T01:18:30,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,090 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:18:30,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:18:30,094 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T01:18:30,095 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:18:30,096 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41905 2024-12-08T01:18:30,098 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41905 connecting to ZooKeeper ensemble=127.0.0.1:52656 2024-12-08T01:18:30,099 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:30,101 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:30,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419050x0, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:18:30,131 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:419050x0, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:18:30,131 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41905-0x1000300bb2c0001 connected 2024-12-08T01:18:30,135 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T01:18:30,144 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T01:18:30,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T01:18:30,151 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:18:30,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41905 2024-12-08T01:18:30,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41905 2024-12-08T01:18:30,152 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41905 2024-12-08T01:18:30,152 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41905 2024-12-08T01:18:30,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41905 2024-12-08T01:18:30,168 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/96982154c400:0 server-side Connection retries=45 2024-12-08T01:18:30,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,168 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:18:30,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:18:30,169 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T01:18:30,169 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:18:30,170 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36455 2024-12-08T01:18:30,171 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36455 connecting to ZooKeeper ensemble=127.0.0.1:52656 2024-12-08T01:18:30,172 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:30,174 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:30,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364550x0, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:18:30,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:364550x0, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:18:30,192 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36455-0x1000300bb2c0002 connected 2024-12-08T01:18:30,192 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T01:18:30,193 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T01:18:30,194 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T01:18:30,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:18:30,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36455 2024-12-08T01:18:30,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36455 2024-12-08T01:18:30,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36455 2024-12-08T01:18:30,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36455 2024-12-08T01:18:30,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36455 2024-12-08T01:18:30,217 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/96982154c400:0 server-side Connection retries=45 2024-12-08T01:18:30,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,217 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:18:30,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:18:30,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:18:30,218 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T01:18:30,218 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:18:30,218 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35607 2024-12-08T01:18:30,220 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35607 connecting to ZooKeeper ensemble=127.0.0.1:52656 2024-12-08T01:18:30,221 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:30,223 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:30,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:356070x0, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:18:30,236 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:356070x0, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:18:30,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35607-0x1000300bb2c0003 connected 2024-12-08T01:18:30,236 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T01:18:30,237 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T01:18:30,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T01:18:30,240 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:18:30,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35607 2024-12-08T01:18:30,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35607 2024-12-08T01:18:30,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35607 2024-12-08T01:18:30,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35607 2024-12-08T01:18:30,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35607 2024-12-08T01:18:30,258 DEBUG [M:0;96982154c400:33573 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;96982154c400:33573 2024-12-08T01:18:30,259 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/96982154c400,33573,1733620709230 2024-12-08T01:18:30,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:18:30,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:18:30,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:18:30,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:18:30,277 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/96982154c400,33573,1733620709230 2024-12-08T01:18:30,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T01:18:30,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T01:18:30,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T01:18:30,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:30,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:30,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:30,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:30,307 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T01:18:30,309 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/96982154c400,33573,1733620709230 from backup master directory 2024-12-08T01:18:30,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/96982154c400,33573,1733620709230 2024-12-08T01:18:30,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:18:30,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:18:30,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:18:30,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:18:30,320 WARN [master/96982154c400:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:18:30,320 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=96982154c400,33573,1733620709230 2024-12-08T01:18:30,323 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T01:18:30,324 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T01:18:30,376 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/hbase.id] with ID: ed8c269a-5f9d-4013-89a3-831bf8295dbd 2024-12-08T01:18:30,376 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/.tmp/hbase.id 2024-12-08T01:18:30,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741826_1002 (size=42) 2024-12-08T01:18:30,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741826_1002 (size=42) 2024-12-08T01:18:30,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741826_1002 (size=42) 2024-12-08T01:18:30,391 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/.tmp/hbase.id]:[hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/hbase.id] 2024-12-08T01:18:30,437 INFO [master/96982154c400:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:18:30,441 INFO [master/96982154c400:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T01:18:30,457 INFO [master/96982154c400:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-08T01:18:30,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:30,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:30,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:30,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:30,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741827_1003 (size=196) 2024-12-08T01:18:30,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741827_1003 (size=196) 2024-12-08T01:18:30,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741827_1003 (size=196) 2024-12-08T01:18:30,502 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:18:30,505 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T01:18:30,510 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:30,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741828_1004 (size=1189) 2024-12-08T01:18:30,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741828_1004 (size=1189) 2024-12-08T01:18:30,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741828_1004 (size=1189) 2024-12-08T01:18:30,557 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store 2024-12-08T01:18:30,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741829_1005 (size=34) 2024-12-08T01:18:30,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741829_1005 (size=34) 2024-12-08T01:18:30,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741829_1005 (size=34) 2024-12-08T01:18:30,581 INFO [master/96982154c400:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T01:18:30,585 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:30,586 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T01:18:30,587 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:18:30,587 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:18:30,589 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T01:18:30,590 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:18:30,590 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:18:30,591 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733620710586Disabling compacts and flushes for region at 1733620710586Disabling writes for close at 1733620710589 (+3 ms)Writing region close event to WAL at 1733620710590 (+1 ms)Closed at 1733620710590 2024-12-08T01:18:30,593 WARN [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/.initializing 2024-12-08T01:18:30,594 DEBUG [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/WALs/96982154c400,33573,1733620709230 2024-12-08T01:18:30,602 INFO [master/96982154c400:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:30,616 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C33573%2C1733620709230, suffix=, logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/WALs/96982154c400,33573,1733620709230, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/oldWALs, maxLogs=10 2024-12-08T01:18:30,649 DEBUG [master/96982154c400:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/WALs/96982154c400,33573,1733620709230/96982154c400%2C33573%2C1733620709230.1733620710621, exclude list is [], retry=0 2024-12-08T01:18:30,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:30,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:30,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:30,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:30,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T01:18:30,708 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/WALs/96982154c400,33573,1733620709230/96982154c400%2C33573%2C1733620709230.1733620710621 2024-12-08T01:18:30,709 DEBUG [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:30,710 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:30,710 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:30,713 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,713 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T01:18:30,768 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:30,771 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:30,771 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,775 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T01:18:30,775 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:30,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:30,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,778 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T01:18:30,778 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:30,779 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:30,779 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T01:18:30,781 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:30,782 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:30,783 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,786 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,787 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,792 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,792 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,797 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:18:30,800 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:18:30,805 DEBUG [master/96982154c400:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:30,806 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73729325, jitterRate=0.09865255653858185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:18:30,811 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733620710724Initializing all the Stores at 1733620710726 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620710726Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620710727 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620710727Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620710727Cleaning up temporary data from old regions at 1733620710793 (+66 ms)Region opened successfully at 1733620710811 (+18 ms) 2024-12-08T01:18:30,812 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T01:18:30,843 DEBUG [master/96982154c400:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b39ed34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:18:30,870 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T01:18:30,880 INFO [master/96982154c400:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T01:18:30,880 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T01:18:30,882 INFO [master/96982154c400:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T01:18:30,883 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T01:18:30,887 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-08T01:18:30,888 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T01:18:30,909 INFO [master/96982154c400:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T01:18:30,917 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T01:18:30,982 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T01:18:30,986 INFO [master/96982154c400:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T01:18:30,989 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T01:18:31,001 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T01:18:31,004 INFO [master/96982154c400:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T01:18:31,007 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T01:18:31,013 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T01:18:31,015 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T01:18:31,024 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T01:18:31,043 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T01:18:31,056 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T01:18:31,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:18:31,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:18:31,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:18:31,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:18:31,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,069 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=96982154c400,33573,1733620709230, sessionid=0x1000300bb2c0000, setting cluster-up flag (Was=false) 2024-12-08T01:18:31,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,130 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T01:18:31,133 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=96982154c400,33573,1733620709230 2024-12-08T01:18:31,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,193 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T01:18:31,196 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=96982154c400,33573,1733620709230 2024-12-08T01:18:31,205 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T01:18:31,246 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(746): ClusterId : ed8c269a-5f9d-4013-89a3-831bf8295dbd 2024-12-08T01:18:31,246 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(746): ClusterId : ed8c269a-5f9d-4013-89a3-831bf8295dbd 2024-12-08T01:18:31,246 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(746): ClusterId : ed8c269a-5f9d-4013-89a3-831bf8295dbd 2024-12-08T01:18:31,249 DEBUG [RS:2;96982154c400:35607 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T01:18:31,249 DEBUG [RS:0;96982154c400:41905 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T01:18:31,249 DEBUG [RS:1;96982154c400:36455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T01:18:31,268 DEBUG [RS:2;96982154c400:35607 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T01:18:31,268 DEBUG [RS:1;96982154c400:36455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T01:18:31,268 DEBUG [RS:0;96982154c400:41905 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T01:18:31,268 DEBUG [RS:2;96982154c400:35607 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T01:18:31,268 DEBUG [RS:1;96982154c400:36455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T01:18:31,268 DEBUG [RS:0;96982154c400:41905 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T01:18:31,271 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T01:18:31,281 INFO [master/96982154c400:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T01:18:31,287 INFO [master/96982154c400:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T01:18:31,288 DEBUG [RS:2;96982154c400:35607 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T01:18:31,288 DEBUG [RS:0;96982154c400:41905 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T01:18:31,288 DEBUG [RS:1;96982154c400:36455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T01:18:31,289 DEBUG [RS:2;96982154c400:35607 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677b6bab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:18:31,289 DEBUG [RS:1;96982154c400:36455 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad480b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:18:31,289 DEBUG [RS:0;96982154c400:41905 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@701bec5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:18:31,294 DEBUG [master/96982154c400:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 96982154c400,33573,1733620709230 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T01:18:31,302 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:18:31,302 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:18:31,302 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:18:31,303 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:18:31,303 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/96982154c400:0, corePoolSize=10, maxPoolSize=10 2024-12-08T01:18:31,303 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,303 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:18:31,303 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,308 DEBUG [RS:0;96982154c400:41905 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;96982154c400:41905 2024-12-08T01:18:31,308 DEBUG [RS:2;96982154c400:35607 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;96982154c400:35607 2024-12-08T01:18:31,308 DEBUG [RS:1;96982154c400:36455 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;96982154c400:36455 2024-12-08T01:18:31,310 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:18:31,310 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T01:18:31,311 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733620741311 2024-12-08T01:18:31,312 INFO [RS:2;96982154c400:35607 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T01:18:31,312 INFO [RS:0;96982154c400:41905 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T01:18:31,312 INFO [RS:1;96982154c400:36455 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T01:18:31,312 INFO [RS:1;96982154c400:36455 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T01:18:31,312 INFO [RS:2;96982154c400:35607 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T01:18:31,312 INFO [RS:0;96982154c400:41905 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T01:18:31,312 DEBUG [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T01:18:31,312 DEBUG [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T01:18:31,312 DEBUG [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T01:18:31,313 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T01:18:31,314 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T01:18:31,315 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(2659): reportForDuty to master=96982154c400,33573,1733620709230 with port=36455, startcode=1733620710167 2024-12-08T01:18:31,315 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(2659): reportForDuty to master=96982154c400,33573,1733620709230 with port=35607, startcode=1733620710216 2024-12-08T01:18:31,315 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(2659): reportForDuty to master=96982154c400,33573,1733620709230 with port=41905, startcode=1733620710058 2024-12-08T01:18:31,318 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:31,318 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T01:18:31,319 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T01:18:31,318 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T01:18:31,319 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T01:18:31,319 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T01:18:31,320 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,325 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T01:18:31,327 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T01:18:31,327 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T01:18:31,331 DEBUG [RS:1;96982154c400:36455 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T01:18:31,331 DEBUG [RS:0;96982154c400:41905 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T01:18:31,331 DEBUG [RS:2;96982154c400:35607 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T01:18:31,332 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T01:18:31,333 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T01:18:31,337 DEBUG [master/96982154c400:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620711334,5,FailOnTimeoutGroup] 2024-12-08T01:18:31,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741831_1007 (size=1321) 2024-12-08T01:18:31,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741831_1007 (size=1321) 2024-12-08T01:18:31,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741831_1007 (size=1321) 2024-12-08T01:18:31,343 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T01:18:31,343 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc 2024-12-08T01:18:31,345 DEBUG [master/96982154c400:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620711338,5,FailOnTimeoutGroup] 2024-12-08T01:18:31,345 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,345 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T01:18:31,346 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,346 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741832_1008 (size=32) 2024-12-08T01:18:31,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741832_1008 (size=32) 2024-12-08T01:18:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741832_1008 (size=32) 2024-12-08T01:18:31,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:31,400 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T01:18:31,402 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34643, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T01:18:31,402 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53685, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T01:18:31,402 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56103, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T01:18:31,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T01:18:31,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:31,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:31,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T01:18:31,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T01:18:31,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:31,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:31,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T01:18:31,410 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33573 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 96982154c400,35607,1733620710216 2024-12-08T01:18:31,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T01:18:31,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:31,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:31,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T01:18:31,413 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33573 {}] master.ServerManager(517): Registering regionserver=96982154c400,35607,1733620710216 2024-12-08T01:18:31,414 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T01:18:31,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:31,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:31,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T01:18:31,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740 2024-12-08T01:18:31,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740 2024-12-08T01:18:31,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T01:18:31,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T01:18:31,422 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:18:31,424 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33573 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 96982154c400,36455,1733620710167 2024-12-08T01:18:31,424 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33573 {}] master.ServerManager(517): Registering regionserver=96982154c400,36455,1733620710167 2024-12-08T01:18:31,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T01:18:31,428 DEBUG [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc 2024-12-08T01:18:31,428 DEBUG [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46839 2024-12-08T01:18:31,428 DEBUG [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T01:18:31,429 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33573 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 96982154c400,41905,1733620710058 2024-12-08T01:18:31,429 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33573 {}] master.ServerManager(517): Registering regionserver=96982154c400,41905,1733620710058 2024-12-08T01:18:31,430 DEBUG [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc 2024-12-08T01:18:31,430 DEBUG [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46839 2024-12-08T01:18:31,430 DEBUG [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T01:18:31,431 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:31,433 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69110508, jitterRate=0.029826819896697998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:18:31,433 DEBUG [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc 2024-12-08T01:18:31,433 DEBUG [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46839 2024-12-08T01:18:31,433 DEBUG [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T01:18:31,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733620711397Initializing all the Stores at 1733620711399 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620711399Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620711400 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620711400Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620711400Cleaning up temporary data from old regions at 1733620711421 (+21 ms)Region opened successfully at 1733620711435 (+14 ms) 2024-12-08T01:18:31,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T01:18:31,435 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T01:18:31,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T01:18:31,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T01:18:31,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T01:18:31,437 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T01:18:31,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733620711435Disabling compacts and flushes for region at 1733620711435Disabling writes for close at 1733620711435Writing region close event to WAL at 1733620711436 (+1 ms)Closed at 1733620711437 (+1 ms) 2024-12-08T01:18:31,440 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:18:31,440 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T01:18:31,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T01:18:31,454 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:18:31,457 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T01:18:31,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:18:31,498 DEBUG [RS:2;96982154c400:35607 {}] zookeeper.ZKUtil(111): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/96982154c400,35607,1733620710216 2024-12-08T01:18:31,498 DEBUG [RS:1;96982154c400:36455 {}] zookeeper.ZKUtil(111): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/96982154c400,36455,1733620710167 2024-12-08T01:18:31,499 WARN [RS:2;96982154c400:35607 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:18:31,499 WARN [RS:1;96982154c400:36455 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:18:31,499 INFO [RS:1;96982154c400:36455 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:31,499 INFO [RS:2;96982154c400:35607 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:31,499 DEBUG [RS:0;96982154c400:41905 {}] zookeeper.ZKUtil(111): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/96982154c400,41905,1733620710058 2024-12-08T01:18:31,499 WARN [RS:0;96982154c400:41905 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:18:31,499 DEBUG [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216 2024-12-08T01:18:31,499 DEBUG [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,36455,1733620710167 2024-12-08T01:18:31,499 INFO [RS:0;96982154c400:41905 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:31,499 DEBUG [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,41905,1733620710058 2024-12-08T01:18:31,501 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [96982154c400,35607,1733620710216] 2024-12-08T01:18:31,501 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [96982154c400,36455,1733620710167] 2024-12-08T01:18:31,501 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [96982154c400,41905,1733620710058] 2024-12-08T01:18:31,522 INFO [RS:1;96982154c400:36455 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T01:18:31,522 INFO [RS:0;96982154c400:41905 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T01:18:31,522 INFO [RS:2;96982154c400:35607 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T01:18:31,533 INFO [RS:1;96982154c400:36455 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T01:18:31,533 INFO [RS:2;96982154c400:35607 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T01:18:31,533 INFO [RS:0;96982154c400:41905 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T01:18:31,538 INFO [RS:0;96982154c400:41905 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:18:31,538 INFO [RS:1;96982154c400:36455 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:18:31,538 INFO [RS:2;96982154c400:35607 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:18:31,538 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,538 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,538 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,539 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T01:18:31,539 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T01:18:31,539 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T01:18:31,545 INFO [RS:1;96982154c400:36455 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T01:18:31,545 INFO [RS:2;96982154c400:35607 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T01:18:31,545 INFO [RS:0;96982154c400:41905 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T01:18:31,547 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,547 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,547 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,547 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,547 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:18:31,548 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:18:31,548 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:18:31,548 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,548 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,549 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,549 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:18:31,549 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,549 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,549 DEBUG [RS:1;96982154c400:36455 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:18:31,549 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:18:31,549 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:18:31,549 DEBUG [RS:0;96982154c400:41905 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:18:31,549 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:18:31,549 DEBUG [RS:2;96982154c400:35607 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:18:31,551 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,551 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,551 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,551 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,551 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,551 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,36455,1733620710167-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:18:31,552 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,552 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,552 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,552 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,552 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,552 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,41905,1733620710058-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:18:31,552 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,552 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,553 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,553 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,553 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,553 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,35607,1733620710216-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:18:31,569 INFO [RS:1;96982154c400:36455 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T01:18:31,571 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,36455,1733620710167-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,572 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,572 INFO [RS:1;96982154c400:36455 {}] regionserver.Replication(171): 96982154c400,36455,1733620710167 started 2024-12-08T01:18:31,579 INFO [RS:0;96982154c400:41905 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T01:18:31,579 INFO [RS:2;96982154c400:35607 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T01:18:31,579 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,41905,1733620710058-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,579 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,35607,1733620710216-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,579 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,579 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,579 INFO [RS:2;96982154c400:35607 {}] regionserver.Replication(171): 96982154c400,35607,1733620710216 started 2024-12-08T01:18:31,579 INFO [RS:0;96982154c400:41905 {}] regionserver.Replication(171): 96982154c400,41905,1733620710058 started 2024-12-08T01:18:31,589 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,589 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1482): Serving as 96982154c400,36455,1733620710167, RpcServer on 96982154c400/172.17.0.2:36455, sessionid=0x1000300bb2c0002 2024-12-08T01:18:31,590 DEBUG [RS:1;96982154c400:36455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T01:18:31,591 DEBUG [RS:1;96982154c400:36455 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 96982154c400,36455,1733620710167 2024-12-08T01:18:31,591 DEBUG [RS:1;96982154c400:36455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,36455,1733620710167' 2024-12-08T01:18:31,591 DEBUG [RS:1;96982154c400:36455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T01:18:31,592 DEBUG [RS:1;96982154c400:36455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T01:18:31,593 DEBUG [RS:1;96982154c400:36455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T01:18:31,593 DEBUG [RS:1;96982154c400:36455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T01:18:31,593 DEBUG [RS:1;96982154c400:36455 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 96982154c400,36455,1733620710167 2024-12-08T01:18:31,594 DEBUG [RS:1;96982154c400:36455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,36455,1733620710167' 2024-12-08T01:18:31,594 DEBUG [RS:1;96982154c400:36455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T01:18:31,594 DEBUG [RS:1;96982154c400:36455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T01:18:31,595 DEBUG [RS:1;96982154c400:36455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T01:18:31,595 INFO [RS:1;96982154c400:36455 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T01:18:31,596 INFO [RS:1;96982154c400:36455 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T01:18:31,601 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,601 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:31,601 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1482): Serving as 96982154c400,41905,1733620710058, RpcServer on 96982154c400/172.17.0.2:41905, sessionid=0x1000300bb2c0001 2024-12-08T01:18:31,601 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1482): Serving as 96982154c400,35607,1733620710216, RpcServer on 96982154c400/172.17.0.2:35607, sessionid=0x1000300bb2c0003 2024-12-08T01:18:31,602 DEBUG [RS:2;96982154c400:35607 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T01:18:31,602 DEBUG [RS:0;96982154c400:41905 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T01:18:31,602 DEBUG [RS:2;96982154c400:35607 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 96982154c400,35607,1733620710216 2024-12-08T01:18:31,602 DEBUG [RS:0;96982154c400:41905 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 96982154c400,41905,1733620710058 2024-12-08T01:18:31,602 DEBUG [RS:2;96982154c400:35607 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,35607,1733620710216' 2024-12-08T01:18:31,602 DEBUG [RS:0;96982154c400:41905 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,41905,1733620710058' 2024-12-08T01:18:31,602 DEBUG [RS:2;96982154c400:35607 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T01:18:31,602 DEBUG [RS:0;96982154c400:41905 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T01:18:31,603 DEBUG [RS:2;96982154c400:35607 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T01:18:31,603 DEBUG [RS:0;96982154c400:41905 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T01:18:31,604 DEBUG [RS:2;96982154c400:35607 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T01:18:31,604 DEBUG [RS:0;96982154c400:41905 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T01:18:31,604 DEBUG [RS:2;96982154c400:35607 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T01:18:31,604 DEBUG [RS:0;96982154c400:41905 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T01:18:31,604 DEBUG [RS:2;96982154c400:35607 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 96982154c400,35607,1733620710216 2024-12-08T01:18:31,604 DEBUG [RS:0;96982154c400:41905 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 96982154c400,41905,1733620710058 2024-12-08T01:18:31,604 DEBUG [RS:0;96982154c400:41905 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,41905,1733620710058' 2024-12-08T01:18:31,604 DEBUG [RS:2;96982154c400:35607 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,35607,1733620710216' 2024-12-08T01:18:31,604 DEBUG [RS:2;96982154c400:35607 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T01:18:31,604 DEBUG [RS:0;96982154c400:41905 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T01:18:31,605 DEBUG [RS:0;96982154c400:41905 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T01:18:31,605 DEBUG [RS:2;96982154c400:35607 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T01:18:31,605 DEBUG [RS:0;96982154c400:41905 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T01:18:31,605 DEBUG [RS:2;96982154c400:35607 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T01:18:31,605 INFO [RS:0;96982154c400:41905 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T01:18:31,605 INFO [RS:2;96982154c400:35607 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T01:18:31,605 INFO [RS:0;96982154c400:41905 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T01:18:31,605 INFO [RS:2;96982154c400:35607 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T01:18:31,608 WARN [96982154c400:33573 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T01:18:31,703 INFO [RS:1;96982154c400:36455 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:31,707 INFO [RS:0;96982154c400:41905 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:31,707 INFO [RS:2;96982154c400:35607 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:31,708 INFO [RS:1;96982154c400:36455 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C36455%2C1733620710167, suffix=, logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,36455,1733620710167, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs, maxLogs=32 2024-12-08T01:18:31,710 INFO [RS:2;96982154c400:35607 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C35607%2C1733620710216, suffix=, logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs, maxLogs=32 2024-12-08T01:18:31,710 INFO [RS:0;96982154c400:41905 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C41905%2C1733620710058, suffix=, logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,41905,1733620710058, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs, maxLogs=32 2024-12-08T01:18:31,725 DEBUG [RS:1;96982154c400:36455 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,36455,1733620710167/96982154c400%2C36455%2C1733620710167.1733620711711, exclude list is [], retry=0 2024-12-08T01:18:31,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:31,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:31,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:31,733 DEBUG [RS:0;96982154c400:41905 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,41905,1733620710058/96982154c400%2C41905%2C1733620710058.1733620711713, exclude list is [], retry=0 2024-12-08T01:18:31,733 INFO [RS:1;96982154c400:36455 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,36455,1733620710167/96982154c400%2C36455%2C1733620710167.1733620711711 2024-12-08T01:18:31,733 DEBUG [RS:2;96982154c400:35607 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216/96982154c400%2C35607%2C1733620710216.1733620711713, exclude list is [], retry=0 2024-12-08T01:18:31,737 DEBUG [RS:1;96982154c400:36455 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:18:31,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:31,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:31,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:31,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:31,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:31,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:31,753 INFO [RS:2;96982154c400:35607 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216/96982154c400%2C35607%2C1733620710216.1733620711713 2024-12-08T01:18:31,753 INFO [RS:0;96982154c400:41905 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,41905,1733620710058/96982154c400%2C41905%2C1733620710058.1733620711713 2024-12-08T01:18:31,757 DEBUG [RS:2;96982154c400:35607 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:18:31,757 DEBUG [RS:0;96982154c400:41905 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:18:31,863 DEBUG [96982154c400:33573 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-08T01:18:31,871 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(204): Hosts are {96982154c400=0} racks are {/default-rack=0} 2024-12-08T01:18:31,877 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T01:18:31,877 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T01:18:31,877 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-08T01:18:31,878 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T01:18:31,878 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T01:18:31,878 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-08T01:18:31,878 INFO [96982154c400:33573 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T01:18:31,878 INFO [96982154c400:33573 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T01:18:31,878 INFO [96982154c400:33573 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-08T01:18:31,878 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T01:18:31,884 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=96982154c400,35607,1733620710216 2024-12-08T01:18:31,891 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,35607,1733620710216, state=OPENING 2024-12-08T01:18:31,930 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T01:18:31,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:18:31,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:31,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:31,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:31,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:31,946 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:18:31,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,35607,1733620710216}] 2024-12-08T01:18:32,124 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:18:32,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36705, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:18:32,138 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T01:18:32,139 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:32,139 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T01:18:32,142 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C35607%2C1733620710216.meta, suffix=.meta, logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs, maxLogs=32 2024-12-08T01:18:32,157 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, exclude list is [], retry=0 2024-12-08T01:18:32,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:32,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:32,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:32,164 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta 2024-12-08T01:18:32,164 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:32,165 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:32,166 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T01:18:32,169 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T01:18:32,174 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T01:18:32,179 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T01:18:32,179 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:32,180 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T01:18:32,180 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T01:18:32,183 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T01:18:32,185 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T01:18:32,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:32,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:32,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T01:18:32,188 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T01:18:32,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:32,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:32,189 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T01:18:32,190 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T01:18:32,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:32,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:32,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T01:18:32,192 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T01:18:32,193 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:32,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:32,194 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T01:18:32,195 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740 2024-12-08T01:18:32,198 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740 2024-12-08T01:18:32,201 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T01:18:32,201 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T01:18:32,202 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:18:32,205 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T01:18:32,207 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70769004, jitterRate=0.05454033613204956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:18:32,207 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T01:18:32,208 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733620712180Writing region info on filesystem at 1733620712180Initializing all the Stores at 1733620712182 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620712182Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620712183 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620712183Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620712183Cleaning up temporary data from old regions at 1733620712201 (+18 ms)Running coprocessor post-open hooks at 1733620712207 (+6 ms)Region opened successfully at 1733620712208 (+1 ms) 2024-12-08T01:18:32,215 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733620712116 2024-12-08T01:18:32,226 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T01:18:32,226 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T01:18:32,228 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,35607,1733620710216 2024-12-08T01:18:32,230 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,35607,1733620710216, state=OPEN 2024-12-08T01:18:32,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:32,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:32,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:32,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:32,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:32,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:32,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:32,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:32,244 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=96982154c400,35607,1733620710216 2024-12-08T01:18:32,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T01:18:32,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,35607,1733620710216 in 296 msec 2024-12-08T01:18:32,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T01:18:32,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 806 msec 2024-12-08T01:18:32,259 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:18:32,259 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T01:18:32,299 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:18:32,301 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1] 2024-12-08T01:18:32,330 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:18:32,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56899, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:18:32,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1190 sec 2024-12-08T01:18:32,354 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733620712354, completionTime=-1 2024-12-08T01:18:32,357 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-08T01:18:32,357 DEBUG [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T01:18:32,391 INFO [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-08T01:18:32,391 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733620772391 2024-12-08T01:18:32,391 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733620832391 2024-12-08T01:18:32,391 INFO [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-12-08T01:18:32,393 DEBUG [master/96982154c400:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-08T01:18:32,400 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,33573,1733620709230-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:32,401 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,33573,1733620709230-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:32,401 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,33573,1733620709230-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:32,403 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-96982154c400:33573, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:32,403 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:32,403 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:32,409 DEBUG [master/96982154c400:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T01:18:32,430 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.109sec 2024-12-08T01:18:32,431 INFO [master/96982154c400:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T01:18:32,432 INFO [master/96982154c400:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T01:18:32,433 INFO [master/96982154c400:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T01:18:32,433 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T01:18:32,434 INFO [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T01:18:32,434 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,33573,1733620709230-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:18:32,435 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,33573,1733620709230-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T01:18:32,439 DEBUG [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T01:18:32,440 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T01:18:32,440 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,33573,1733620709230-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:32,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70faddef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:18:32,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,33573,-1 for getting cluster id 2024-12-08T01:18:32,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:18:32,468 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ed8c269a-5f9d-4013-89a3-831bf8295dbd' 2024-12-08T01:18:32,471 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:18:32,471 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ed8c269a-5f9d-4013-89a3-831bf8295dbd" 2024-12-08T01:18:32,472 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457d232a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:18:32,472 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,33573,-1] 2024-12-08T01:18:32,474 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:18:32,476 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:18:32,477 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:18:32,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a92b38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:18:32,480 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:18:32,487 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1] 2024-12-08T01:18:32,487 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:18:32,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:18:32,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=96982154c400,33573,1733620709230 2024-12-08T01:18:32,511 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:46839/hbase 2024-12-08T01:18:32,522 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=366, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=10564 2024-12-08T01:18:32,538 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:32,541 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:32,542 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:32,547 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39272547, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-39272547, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:32,565 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39272547/hregion-39272547.1733620712547, exclude list is [], retry=0 2024-12-08T01:18:32,569 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:32,569 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:32,570 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:32,573 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39272547/hregion-39272547.1733620712547 2024-12-08T01:18:32,574 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:32,574 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d564946f14fd75ff3168032916db8a82, NAME => 'testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:32,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741838_1014 (size=64) 2024-12-08T01:18:32,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741838_1014 (size=64) 2024-12-08T01:18:32,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741838_1014 (size=64) 2024-12-08T01:18:32,588 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:32,592 INFO [StoreOpener-d564946f14fd75ff3168032916db8a82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,594 INFO [StoreOpener-d564946f14fd75ff3168032916db8a82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d564946f14fd75ff3168032916db8a82 columnFamilyName a 2024-12-08T01:18:32,594 DEBUG [StoreOpener-d564946f14fd75ff3168032916db8a82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:32,595 INFO [StoreOpener-d564946f14fd75ff3168032916db8a82-1 {}] regionserver.HStore(327): Store=d564946f14fd75ff3168032916db8a82/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:32,596 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,598 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,598 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,601 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,605 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:32,606 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d564946f14fd75ff3168032916db8a82; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75344550, jitterRate=0.12272128462791443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:32,608 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d564946f14fd75ff3168032916db8a82: Writing region info on filesystem at 1733620712588Initializing all the Stores at 1733620712590 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620712590Cleaning up temporary data from old regions at 1733620712598 (+8 ms)Region opened successfully at 1733620712608 (+10 ms) 2024-12-08T01:18:32,608 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d564946f14fd75ff3168032916db8a82, disabling compactions & flushes 2024-12-08T01:18:32,608 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82. 2024-12-08T01:18:32,608 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82. 2024-12-08T01:18:32,608 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82. after waiting 0 ms 2024-12-08T01:18:32,608 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82. 2024-12-08T01:18:32,609 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82. 2024-12-08T01:18:32,609 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d564946f14fd75ff3168032916db8a82: Waiting for close lock at 1733620712608Disabling compacts and flushes for region at 1733620712608Disabling writes for close at 1733620712608Writing region close event to WAL at 1733620712609 (+1 ms)Closed at 1733620712609 2024-12-08T01:18:32,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741837_1013 (size=93) 2024-12-08T01:18:32,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741837_1013 (size=93) 2024-12-08T01:18:32,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741837_1013 (size=93) 2024-12-08T01:18:32,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:32,621 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39272547:(num 1733620712547) 2024-12-08T01:18:32,622 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-08T01:18:32,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741839_1015 (size=369) 2024-12-08T01:18:32,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741839_1015 (size=369) 2024-12-08T01:18:32,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741839_1015 (size=369) 2024-12-08T01:18:32,640 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-08T01:18:32,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741840_1016 (size=231) 2024-12-08T01:18:32,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741840_1016 (size=231) 2024-12-08T01:18:32,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741840_1016 (size=231) 2024-12-08T01:18:32,671 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1, size=369 (369bytes) 2024-12-08T01:18:32,672 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-08T01:18:32,672 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-08T01:18:32,672 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1 2024-12-08T01:18:32,677 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1 after 3ms 2024-12-08T01:18:32,685 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1 took 16ms 2024-12-08T01:18:32,690 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1 so closing down 2024-12-08T01:18:32,694 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-08T01:18:32,695 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:32,696 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000001-wal-1.temp 2024-12-08T01:18:32,697 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:32,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741841_1017 (size=369) 2024-12-08T01:18:32,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741841_1017 (size=369) 2024-12-08T01:18:32,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741841_1017 (size=369) 2024-12-08T01:18:32,708 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:32,711 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002 2024-12-08T01:18:32,714 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-08T01:18:32,714 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1, journal: Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1, size=369 (369bytes) at 1733620712671Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1 so closing down at 1733620712690 (+19 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000001-wal-1.temp at 1733620712696 (+6 ms)3 split writer threads finished at 1733620712697 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733620712708 (+11 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002 at 1733620712711 (+3 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733620712714 (+3 ms) 2024-12-08T01:18:32,729 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2, size=231 (231bytes) 2024-12-08T01:18:32,729 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2 2024-12-08T01:18:32,730 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2 after 0ms 2024-12-08T01:18:32,733 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2 took 5ms 2024-12-08T01:18:32,735 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2 so closing down 2024-12-08T01:18:32,735 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:32,737 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-08T01:18:32,739 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002-wal-2.temp 2024-12-08T01:18:32,739 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:32,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741842_1018 (size=231) 2024-12-08T01:18:32,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741842_1018 (size=231) 2024-12-08T01:18:32,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741842_1018 (size=231) 2024-12-08T01:18:32,748 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:32,753 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-12-08T01:18:32,755 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-08T01:18:32,755 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2, journal: Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2, size=231 (231bytes) at 1733620712729Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2 so closing down at 1733620712735 (+6 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002-wal-2.temp at 1733620712739 (+4 ms)3 split writer threads finished at 1733620712739Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733620712748 (+9 ms)Processed 1 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733620712755 (+7 ms) 2024-12-08T01:18:32,755 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:32,758 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:32,770 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal.1733620712758, exclude list is [], retry=0 2024-12-08T01:18:32,774 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:32,775 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:32,775 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:32,777 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal.1733620712758 2024-12-08T01:18:32,778 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:18:32,778 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d564946f14fd75ff3168032916db8a82, NAME => 'testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:32,778 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:32,779 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,779 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,781 INFO [StoreOpener-d564946f14fd75ff3168032916db8a82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,782 INFO [StoreOpener-d564946f14fd75ff3168032916db8a82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d564946f14fd75ff3168032916db8a82 columnFamilyName a 2024-12-08T01:18:32,782 DEBUG [StoreOpener-d564946f14fd75ff3168032916db8a82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:32,784 INFO [StoreOpener-d564946f14fd75ff3168032916db8a82-1 {}] regionserver.HStore(327): Store=d564946f14fd75ff3168032916db8a82/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:32,784 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,785 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,787 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,788 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002 2024-12-08T01:18:32,796 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002 2024-12-08T01:18:32,799 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d564946f14fd75ff3168032916db8a82 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-08T01:18:32,853 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/.tmp/a/07cb1b8abee74e6a89f740ebe46a099e is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733620712621/Put/seqid=0 2024-12-08T01:18:32,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741844_1020 (size=5170) 2024-12-08T01:18:32,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741844_1020 (size=5170) 2024-12-08T01:18:32,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741844_1020 (size=5170) 2024-12-08T01:18:32,866 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/.tmp/a/07cb1b8abee74e6a89f740ebe46a099e 2024-12-08T01:18:32,911 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/.tmp/a/07cb1b8abee74e6a89f740ebe46a099e as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/a/07cb1b8abee74e6a89f740ebe46a099e 2024-12-08T01:18:32,921 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/a/07cb1b8abee74e6a89f740ebe46a099e, entries=2, sequenceid=2, filesize=5.0 K 2024-12-08T01:18:32,927 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d564946f14fd75ff3168032916db8a82 in 126ms, sequenceid=2, compaction requested=false; wal=null 2024-12-08T01:18:32,929 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/0000000000000000002 2024-12-08T01:18:32,930 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,930 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,933 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d564946f14fd75ff3168032916db8a82 2024-12-08T01:18:32,936 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/d564946f14fd75ff3168032916db8a82/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-08T01:18:32,938 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d564946f14fd75ff3168032916db8a82; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62100530, jitterRate=-0.07462999224662781}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:32,939 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d564946f14fd75ff3168032916db8a82: Writing region info on filesystem at 1733620712779Initializing all the Stores at 1733620712780 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620712780Obtaining lock to block concurrent updates at 1733620712799 (+19 ms)Preparing flush snapshotting stores in d564946f14fd75ff3168032916db8a82 at 1733620712799Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733620712802 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733620712538.d564946f14fd75ff3168032916db8a82. at 1733620712802Flushing d564946f14fd75ff3168032916db8a82/a: creating writer at 1733620712804 (+2 ms)Flushing d564946f14fd75ff3168032916db8a82/a: appending metadata at 1733620712845 (+41 ms)Flushing d564946f14fd75ff3168032916db8a82/a: closing flushed file at 1733620712847 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23e86f3c: reopening flushed file at 1733620712909 (+62 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d564946f14fd75ff3168032916db8a82 in 126ms, sequenceid=2, compaction requested=false; wal=null at 1733620712927 (+18 ms)Cleaning up temporary data from old regions at 1733620712930 (+3 ms)Region opened successfully at 1733620712939 (+9 ms) 2024-12-08T01:18:32,960 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=378 (was 366) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:36926 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:43860 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:43682 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:46839/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37594 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37562 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37090 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=273 (was 273), ProcessCount=11 (was 11), AvailableMemoryMB=10551 (was 10564) 2024-12-08T01:18:32,969 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=10550 2024-12-08T01:18:32,983 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:32,985 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:32,985 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:32,988 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62742094, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-62742094, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:32,999 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62742094/hregion-62742094.1733620712989, exclude list is [], retry=0 2024-12-08T01:18:33,003 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:33,003 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:33,003 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:33,006 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62742094/hregion-62742094.1733620712989 2024-12-08T01:18:33,006 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:33,007 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9a26e01f78c0dcf45cc3c19bc016b044, NAME => 'testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:33,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741846_1022 (size=64) 2024-12-08T01:18:33,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741846_1022 (size=64) 2024-12-08T01:18:33,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741846_1022 (size=64) 2024-12-08T01:18:33,021 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:33,023 INFO [StoreOpener-9a26e01f78c0dcf45cc3c19bc016b044-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,025 INFO [StoreOpener-9a26e01f78c0dcf45cc3c19bc016b044-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a26e01f78c0dcf45cc3c19bc016b044 columnFamilyName a 2024-12-08T01:18:33,025 DEBUG [StoreOpener-9a26e01f78c0dcf45cc3c19bc016b044-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:33,026 INFO [StoreOpener-9a26e01f78c0dcf45cc3c19bc016b044-1 {}] regionserver.HStore(327): Store=9a26e01f78c0dcf45cc3c19bc016b044/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:33,026 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,028 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,028 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,030 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,034 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:33,034 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9a26e01f78c0dcf45cc3c19bc016b044; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72957990, jitterRate=0.08715876936912537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:33,035 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9a26e01f78c0dcf45cc3c19bc016b044: Writing region info on filesystem at 1733620713021Initializing all the Stores at 1733620713022 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620713022Cleaning up temporary data from old regions at 1733620713028 (+6 ms)Region opened successfully at 1733620713034 (+6 ms) 2024-12-08T01:18:33,035 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9a26e01f78c0dcf45cc3c19bc016b044, disabling compactions & flushes 2024-12-08T01:18:33,035 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044. 2024-12-08T01:18:33,035 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044. 2024-12-08T01:18:33,035 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044. after waiting 0 ms 2024-12-08T01:18:33,035 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044. 2024-12-08T01:18:33,035 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044. 2024-12-08T01:18:33,035 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9a26e01f78c0dcf45cc3c19bc016b044: Waiting for close lock at 1733620713035Disabling compacts and flushes for region at 1733620713035Disabling writes for close at 1733620713035Writing region close event to WAL at 1733620713035Closed at 1733620713035 2024-12-08T01:18:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741845_1021 (size=93) 2024-12-08T01:18:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741845_1021 (size=93) 2024-12-08T01:18:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741845_1021 (size=93) 2024-12-08T01:18:33,041 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:33,041 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62742094:(num 1733620712989) 2024-12-08T01:18:33,042 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-08T01:18:33,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741847_1023 (size=369) 2024-12-08T01:18:33,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741847_1023 (size=369) 2024-12-08T01:18:33,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741847_1023 (size=369) 2024-12-08T01:18:33,054 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-08T01:18:33,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741848_1024 (size=231) 2024-12-08T01:18:33,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741848_1024 (size=231) 2024-12-08T01:18:33,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741848_1024 (size=231) 2024-12-08T01:18:33,078 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2, size=231 (231bytes) 2024-12-08T01:18:33,078 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2 2024-12-08T01:18:33,079 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2 after 1ms 2024-12-08T01:18:33,082 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2 took 5ms 2024-12-08T01:18:33,084 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2 so closing down 2024-12-08T01:18:33,084 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:33,086 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-08T01:18:33,087 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002-wal-2.temp 2024-12-08T01:18:33,087 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:33,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741849_1025 (size=231) 2024-12-08T01:18:33,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741849_1025 (size=231) 2024-12-08T01:18:33,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741849_1025 (size=231) 2024-12-08T01:18:33,096 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:33,098 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002 2024-12-08T01:18:33,098 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-08T01:18:33,098 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2, journal: Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2, size=231 (231bytes) at 1733620713078Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2 so closing down at 1733620713084 (+6 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002-wal-2.temp at 1733620713087 (+3 ms)3 split writer threads finished at 1733620713087Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733620713096 (+9 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002 at 1733620713098 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733620713098 2024-12-08T01:18:33,112 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1, size=369 (369bytes) 2024-12-08T01:18:33,112 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1 2024-12-08T01:18:33,113 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1 after 1ms 2024-12-08T01:18:33,116 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1 took 4ms 2024-12-08T01:18:33,119 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1 so closing down 2024-12-08T01:18:33,119 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:33,121 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-08T01:18:33,122 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000001-wal-1.temp 2024-12-08T01:18:33,123 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:33,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741850_1026 (size=369) 2024-12-08T01:18:33,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741850_1026 (size=369) 2024-12-08T01:18:33,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741850_1026 (size=369) 2024-12-08T01:18:33,131 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:33,137 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002, length=231 2024-12-08T01:18:33,139 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002 2024-12-08T01:18:33,140 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-08T01:18:33,140 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1, journal: Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1, size=369 (369bytes) at 1733620713112Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1 so closing down at 1733620713119 (+7 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000001-wal-1.temp at 1733620713122 (+3 ms)3 split writer threads finished at 1733620713123 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733620713132 (+9 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002 at 1733620713139 (+7 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733620713140 (+1 ms) 2024-12-08T01:18:33,140 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:33,142 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:33,153 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal.1733620713142, exclude list is [], retry=0 2024-12-08T01:18:33,156 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:33,156 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:33,157 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:33,158 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal.1733620713142 2024-12-08T01:18:33,159 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:33,159 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a26e01f78c0dcf45cc3c19bc016b044, NAME => 'testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:33,159 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:33,159 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,159 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,161 INFO [StoreOpener-9a26e01f78c0dcf45cc3c19bc016b044-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,163 INFO [StoreOpener-9a26e01f78c0dcf45cc3c19bc016b044-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a26e01f78c0dcf45cc3c19bc016b044 columnFamilyName a 2024-12-08T01:18:33,163 DEBUG [StoreOpener-9a26e01f78c0dcf45cc3c19bc016b044-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:33,164 INFO [StoreOpener-9a26e01f78c0dcf45cc3c19bc016b044-1 {}] regionserver.HStore(327): Store=9a26e01f78c0dcf45cc3c19bc016b044/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:33,164 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,165 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,167 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,168 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002 2024-12-08T01:18:33,172 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002 2024-12-08T01:18:33,172 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9a26e01f78c0dcf45cc3c19bc016b044 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-08T01:18:33,187 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/.tmp/a/fa81698b730d416eaa312f38acecd90c is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733620713041/Put/seqid=0 2024-12-08T01:18:33,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741852_1028 (size=5170) 2024-12-08T01:18:33,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741852_1028 (size=5170) 2024-12-08T01:18:33,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741852_1028 (size=5170) 2024-12-08T01:18:33,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/.tmp/a/fa81698b730d416eaa312f38acecd90c 2024-12-08T01:18:33,204 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/.tmp/a/fa81698b730d416eaa312f38acecd90c as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/a/fa81698b730d416eaa312f38acecd90c 2024-12-08T01:18:33,213 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/a/fa81698b730d416eaa312f38acecd90c, entries=2, sequenceid=2, filesize=5.0 K 2024-12-08T01:18:33,213 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 9a26e01f78c0dcf45cc3c19bc016b044 in 41ms, sequenceid=2, compaction requested=false; wal=null 2024-12-08T01:18:33,214 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/0000000000000000002 2024-12-08T01:18:33,214 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,214 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,217 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9a26e01f78c0dcf45cc3c19bc016b044 2024-12-08T01:18:33,219 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a26e01f78c0dcf45cc3c19bc016b044/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-08T01:18:33,221 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9a26e01f78c0dcf45cc3c19bc016b044; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64612490, jitterRate=-0.03719887137413025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:33,221 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9a26e01f78c0dcf45cc3c19bc016b044: Writing region info on filesystem at 1733620713159Initializing all the Stores at 1733620713161 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620713161Obtaining lock to block concurrent updates at 1733620713172 (+11 ms)Preparing flush snapshotting stores in 9a26e01f78c0dcf45cc3c19bc016b044 at 1733620713172Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733620713172Flushing stores of testReplayEditsWrittenIntoWAL,,1733620712983.9a26e01f78c0dcf45cc3c19bc016b044. at 1733620713172Flushing 9a26e01f78c0dcf45cc3c19bc016b044/a: creating writer at 1733620713172Flushing 9a26e01f78c0dcf45cc3c19bc016b044/a: appending metadata at 1733620713186 (+14 ms)Flushing 9a26e01f78c0dcf45cc3c19bc016b044/a: closing flushed file at 1733620713186Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a26362b: reopening flushed file at 1733620713203 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 9a26e01f78c0dcf45cc3c19bc016b044 in 41ms, sequenceid=2, compaction requested=false; wal=null at 1733620713213 (+10 ms)Cleaning up temporary data from old regions at 1733620713214 (+1 ms)Region opened successfully at 1733620713221 (+7 ms) 2024-12-08T01:18:33,240 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:43932 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:36926 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37562 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:43682 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37652 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37186 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=273 (was 273), ProcessCount=11 (was 11), AvailableMemoryMB=10545 (was 10550) 2024-12-08T01:18:33,253 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=10545 2024-12-08T01:18:33,269 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:33,271 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:33,272 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:33,275 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29038195, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-29038195, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:33,286 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29038195/hregion-29038195.1733620713275, exclude list is [], retry=0 2024-12-08T01:18:33,289 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:33,289 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:33,290 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:33,292 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29038195/hregion-29038195.1733620713275 2024-12-08T01:18:33,293 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:33,293 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 27944d03348a945284cb4e659af3bfa8, NAME => 'testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:33,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741854_1030 (size=64) 2024-12-08T01:18:33,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741854_1030 (size=64) 2024-12-08T01:18:33,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741854_1030 (size=64) 2024-12-08T01:18:33,306 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:33,308 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,310 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27944d03348a945284cb4e659af3bfa8 columnFamilyName a 2024-12-08T01:18:33,310 DEBUG [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:33,311 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(327): Store=27944d03348a945284cb4e659af3bfa8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:33,311 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,313 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27944d03348a945284cb4e659af3bfa8 columnFamilyName b 2024-12-08T01:18:33,313 DEBUG [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:33,313 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(327): Store=27944d03348a945284cb4e659af3bfa8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:33,314 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,315 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27944d03348a945284cb4e659af3bfa8 columnFamilyName c 2024-12-08T01:18:33,315 DEBUG [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:33,316 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(327): Store=27944d03348a945284cb4e659af3bfa8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:33,316 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,317 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,317 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,319 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,319 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,319 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:33,320 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:33,323 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:33,323 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 27944d03348a945284cb4e659af3bfa8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63532894, jitterRate=-0.0532861053943634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:33,324 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 27944d03348a945284cb4e659af3bfa8: Writing region info on filesystem at 1733620713307Initializing all the Stores at 1733620713308 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620713308Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620713308Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620713308Cleaning up temporary data from old regions at 1733620713319 (+11 ms)Region opened successfully at 1733620713323 (+4 ms) 2024-12-08T01:18:33,324 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 27944d03348a945284cb4e659af3bfa8, disabling compactions & flushes 2024-12-08T01:18:33,324 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:33,324 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:33,324 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. after waiting 0 ms 2024-12-08T01:18:33,324 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:33,324 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:33,324 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 27944d03348a945284cb4e659af3bfa8: Waiting for close lock at 1733620713324Disabling compacts and flushes for region at 1733620713324Disabling writes for close at 1733620713324Writing region close event to WAL at 1733620713324Closed at 1733620713324 2024-12-08T01:18:33,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741853_1029 (size=93) 2024-12-08T01:18:33,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741853_1029 (size=93) 2024-12-08T01:18:33,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741853_1029 (size=93) 2024-12-08T01:18:33,331 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:33,331 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29038195:(num 1733620713275) 2024-12-08T01:18:33,331 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:33,333 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:33,345 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334, exclude list is [], retry=0 2024-12-08T01:18:33,348 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:33,349 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:33,349 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:33,351 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 2024-12-08T01:18:33,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:33,517 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334, size=0 (0bytes) 2024-12-08T01:18:33,517 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 might be still open, length is 0 2024-12-08T01:18:33,517 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 2024-12-08T01:18:33,518 WARN [IPC Server handler 3 on default port 46839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-08T01:18:33,519 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 after 2ms 2024-12-08T01:18:34,344 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:43984 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39423:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43984 dst: /127.0.0.1:39423 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39423 remote=/127.0.0.1:43984]. Total timeout mills is 60000, 59142 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:34,345 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37688 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46125:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37688 dst: /127.0.0.1:46125 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:34,345 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37218 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37218 dst: /127.0.0.1:45653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:34,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741855_1032 (size=470675) 2024-12-08T01:18:34,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741855_1032 (size=470675) 2024-12-08T01:18:34,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741855_1032 (size=470675) 2024-12-08T01:18:37,521 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 after 4003ms 2024-12-08T01:18:37,530 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 took 4012ms 2024-12-08T01:18:37,535 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733620713334.temp 2024-12-08T01:18:37,540 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000000001-wal.1733620713334.temp 2024-12-08T01:18:37,601 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334; continuing. 2024-12-08T01:18:37,601 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 so closing down 2024-12-08T01:18:37,601 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:37,603 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:37,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741856_1033 (size=470683) 2024-12-08T01:18:37,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741856_1033 (size=470683) 2024-12-08T01:18:37,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741856_1033 (size=470683) 2024-12-08T01:18:37,607 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000000001-wal.1733620713334.temp (wrote 3002 edits, skipped 0 edits in 27 ms) 2024-12-08T01:18:37,608 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000000001-wal.1733620713334.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000003002 2024-12-08T01:18:37,609 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334, size=0, length=0, corrupted=false, cancelled=false 2024-12-08T01:18:37,609 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334, size=0 (0bytes) at 1733620713517Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000000001-wal.1733620713334.temp at 1733620717540 (+4023 ms)Split 1024 edits, skipped 0 edits. at 1733620717573 (+33 ms)Split 2048 edits, skipped 0 edits. at 1733620717589 (+16 ms)Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 so closing down at 1733620717601 (+12 ms)3 split writer threads finished at 1733620717603 (+2 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000000001-wal.1733620713334.temp (wrote 3002 edits, skipped 0 edits in 27 ms) at 1733620717607 (+4 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000000001-wal.1733620713334.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000003002 at 1733620717608 (+1 ms)Processed 3002 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334, size=0, length=0, corrupted=false, cancelled=false at 1733620717609 (+1 ms) 2024-12-08T01:18:37,611 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620713334 2024-12-08T01:18:37,612 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000003002 2024-12-08T01:18:37,612 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:37,615 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:37,627 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620717615, exclude list is [], retry=0 2024-12-08T01:18:37,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:37,631 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:37,631 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:37,633 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620717615 2024-12-08T01:18:37,633 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:37,633 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:37,635 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:37,636 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27944d03348a945284cb4e659af3bfa8 columnFamilyName a 2024-12-08T01:18:37,636 DEBUG [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:37,637 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(327): Store=27944d03348a945284cb4e659af3bfa8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:37,637 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:37,638 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27944d03348a945284cb4e659af3bfa8 columnFamilyName b 2024-12-08T01:18:37,638 DEBUG [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:37,639 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(327): Store=27944d03348a945284cb4e659af3bfa8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:37,639 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:37,640 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27944d03348a945284cb4e659af3bfa8 columnFamilyName c 2024-12-08T01:18:37,640 DEBUG [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:37,640 INFO [StoreOpener-27944d03348a945284cb4e659af3bfa8-1 {}] regionserver.HStore(327): Store=27944d03348a945284cb4e659af3bfa8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:37,640 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:37,641 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:37,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:37,644 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000003002 2024-12-08T01:18:37,677 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-08T01:18:37,691 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:18:37,751 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T01:18:37,994 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 27944d03348a945284cb4e659af3bfa8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-08T01:18:38,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/a/88aab472bc0e41509aa2e173d5eb1b50 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733620713360/Put/seqid=0 2024-12-08T01:18:38,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741858_1035 (size=50463) 2024-12-08T01:18:38,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741858_1035 (size=50463) 2024-12-08T01:18:38,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741858_1035 (size=50463) 2024-12-08T01:18:38,033 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/a/88aab472bc0e41509aa2e173d5eb1b50 2024-12-08T01:18:38,040 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/a/88aab472bc0e41509aa2e173d5eb1b50 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/a/88aab472bc0e41509aa2e173d5eb1b50 2024-12-08T01:18:38,047 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/a/88aab472bc0e41509aa2e173d5eb1b50, entries=754, sequenceid=754, filesize=49.3 K 2024-12-08T01:18:38,048 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 27944d03348a945284cb4e659af3bfa8 in 54ms, sequenceid=754, compaction requested=false; wal=null 2024-12-08T01:18:38,060 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-08T01:18:38,061 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 27944d03348a945284cb4e659af3bfa8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-08T01:18:38,070 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/a/d1de3ed5c2454cb9bbcf79ba3fdd1858 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733620713395/Put/seqid=0 2024-12-08T01:18:38,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741859_1036 (size=20072) 2024-12-08T01:18:38,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741859_1036 (size=20072) 2024-12-08T01:18:38,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741859_1036 (size=20072) 2024-12-08T01:18:38,080 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/a/d1de3ed5c2454cb9bbcf79ba3fdd1858 2024-12-08T01:18:38,105 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/b/cbf8fcb147fc47f88c2e599364f9eb8a is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733620713412/Put/seqid=0 2024-12-08T01:18:38,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741860_1037 (size=35835) 2024-12-08T01:18:38,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741860_1037 (size=35835) 2024-12-08T01:18:38,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741860_1037 (size=35835) 2024-12-08T01:18:38,113 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/b/cbf8fcb147fc47f88c2e599364f9eb8a 2024-12-08T01:18:38,121 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/a/d1de3ed5c2454cb9bbcf79ba3fdd1858 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/a/d1de3ed5c2454cb9bbcf79ba3fdd1858 2024-12-08T01:18:38,127 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/a/d1de3ed5c2454cb9bbcf79ba3fdd1858, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-08T01:18:38,129 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/b/cbf8fcb147fc47f88c2e599364f9eb8a as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/b/cbf8fcb147fc47f88c2e599364f9eb8a 2024-12-08T01:18:38,138 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/b/cbf8fcb147fc47f88c2e599364f9eb8a, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-08T01:18:38,138 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 27944d03348a945284cb4e659af3bfa8 in 78ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-08T01:18:38,152 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-08T01:18:38,153 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 27944d03348a945284cb4e659af3bfa8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-08T01:18:38,160 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/b/d733ea3015204a239bf32c445e5d3da3 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733620713426/Put/seqid=0 2024-12-08T01:18:38,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741861_1038 (size=35082) 2024-12-08T01:18:38,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741861_1038 (size=35082) 2024-12-08T01:18:38,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741861_1038 (size=35082) 2024-12-08T01:18:38,172 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/b/d733ea3015204a239bf32c445e5d3da3 2024-12-08T01:18:38,197 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/c/6580b71246604aff8bbeb37b7e1072b8 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733620713451/Put/seqid=0 2024-12-08T01:18:38,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741862_1039 (size=20825) 2024-12-08T01:18:38,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741862_1039 (size=20825) 2024-12-08T01:18:38,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741862_1039 (size=20825) 2024-12-08T01:18:38,205 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/c/6580b71246604aff8bbeb37b7e1072b8 2024-12-08T01:18:38,213 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/b/d733ea3015204a239bf32c445e5d3da3 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/b/d733ea3015204a239bf32c445e5d3da3 2024-12-08T01:18:38,220 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/b/d733ea3015204a239bf32c445e5d3da3, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-08T01:18:38,221 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/c/6580b71246604aff8bbeb37b7e1072b8 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/c/6580b71246604aff8bbeb37b7e1072b8 2024-12-08T01:18:38,229 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/c/6580b71246604aff8bbeb37b7e1072b8, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-08T01:18:38,230 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 27944d03348a945284cb4e659af3bfa8 in 76ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-08T01:18:38,240 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733620713485/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:38,244 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000003002 2024-12-08T01:18:38,244 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-08T01:18:38,245 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 27944d03348a945284cb4e659af3bfa8 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-08T01:18:38,254 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/c/d37da7df93364b509d6b69093d41cde1 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733620713456/Put/seqid=0 2024-12-08T01:18:38,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741863_1040 (size=50301) 2024-12-08T01:18:38,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741863_1040 (size=50301) 2024-12-08T01:18:38,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741863_1040 (size=50301) 2024-12-08T01:18:38,263 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/c/d37da7df93364b509d6b69093d41cde1 2024-12-08T01:18:38,271 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d37da7df93364b509d6b69093d41cde1 2024-12-08T01:18:38,273 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/.tmp/c/d37da7df93364b509d6b69093d41cde1 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/c/d37da7df93364b509d6b69093d41cde1 2024-12-08T01:18:38,280 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d37da7df93364b509d6b69093d41cde1 2024-12-08T01:18:38,280 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/c/d37da7df93364b509d6b69093d41cde1, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-08T01:18:38,281 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 27944d03348a945284cb4e659af3bfa8 in 36ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-08T01:18:38,282 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/0000000000000003002 2024-12-08T01:18:38,284 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:38,284 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:38,285 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:18:38,287 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 27944d03348a945284cb4e659af3bfa8 2024-12-08T01:18:38,289 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/27944d03348a945284cb4e659af3bfa8/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-08T01:18:38,291 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 27944d03348a945284cb4e659af3bfa8; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69764221, jitterRate=0.03956790268421173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:18:38,291 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 27944d03348a945284cb4e659af3bfa8: Writing region info on filesystem at 1733620717634Initializing all the Stores at 1733620717635 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620717635Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620717635Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620717635Cleaning up temporary data from old regions at 1733620718284 (+649 ms)Region opened successfully at 1733620718291 (+7 ms) 2024-12-08T01:18:38,352 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 27944d03348a945284cb4e659af3bfa8, disabling compactions & flushes 2024-12-08T01:18:38,352 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:38,352 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:38,353 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. after waiting 0 ms 2024-12-08T01:18:38,353 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:38,355 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733620713270.27944d03348a945284cb4e659af3bfa8. 2024-12-08T01:18:38,355 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 27944d03348a945284cb4e659af3bfa8: Waiting for close lock at 1733620718352Disabling compacts and flushes for region at 1733620718352Disabling writes for close at 1733620718353 (+1 ms)Writing region close event to WAL at 1733620718355 (+2 ms)Closed at 1733620718355 2024-12-08T01:18:38,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741857_1034 (size=93) 2024-12-08T01:18:38,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741857_1034 (size=93) 2024-12-08T01:18:38,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741857_1034 (size=93) 2024-12-08T01:18:38,362 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:38,362 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620717615) 2024-12-08T01:18:38,378 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=402 (was 388) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-722768394_22 at /127.0.0.1:37272 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:46839 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-722768394_22 at /127.0.0.1:37764 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34225 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:34225 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:46839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-722768394_22 at /127.0.0.1:37290 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:43303 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43303 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-722768394_22 at /127.0.0.1:44026 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=267 (was 273), ProcessCount=11 (was 11), AvailableMemoryMB=10473 (was 10545) 2024-12-08T01:18:38,388 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=267, ProcessCount=11, AvailableMemoryMB=10471 2024-12-08T01:18:38,403 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:38,405 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:38,406 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:38,409 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80557104, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-80557104, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:38,422 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80557104/hregion-80557104.1733620718410, exclude list is [], retry=0 2024-12-08T01:18:38,426 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:38,426 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:38,426 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:38,428 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80557104/hregion-80557104.1733620718410 2024-12-08T01:18:38,429 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:38,429 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 33e0fd9da81a8a4356ecb1b9d0341ed1, NAME => 'test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:38,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741865_1042 (size=43) 2024-12-08T01:18:38,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741865_1042 (size=43) 2024-12-08T01:18:38,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741865_1042 (size=43) 2024-12-08T01:18:38,440 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:38,441 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,443 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33e0fd9da81a8a4356ecb1b9d0341ed1 columnFamilyName a 2024-12-08T01:18:38,443 DEBUG [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:38,444 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(327): Store=33e0fd9da81a8a4356ecb1b9d0341ed1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:38,444 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,446 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33e0fd9da81a8a4356ecb1b9d0341ed1 columnFamilyName b 2024-12-08T01:18:38,446 DEBUG [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:38,446 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(327): Store=33e0fd9da81a8a4356ecb1b9d0341ed1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:38,447 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,448 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33e0fd9da81a8a4356ecb1b9d0341ed1 columnFamilyName c 2024-12-08T01:18:38,448 DEBUG [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:38,449 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(327): Store=33e0fd9da81a8a4356ecb1b9d0341ed1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:38,449 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,449 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,450 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,451 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,451 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,451 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:38,453 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,455 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:38,456 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33e0fd9da81a8a4356ecb1b9d0341ed1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67464308, jitterRate=0.005296528339385986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:38,457 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33e0fd9da81a8a4356ecb1b9d0341ed1: Writing region info on filesystem at 1733620718440Initializing all the Stores at 1733620718441 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620718441Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620718441Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620718441Cleaning up temporary data from old regions at 1733620718451 (+10 ms)Region opened successfully at 1733620718457 (+6 ms) 2024-12-08T01:18:38,457 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 33e0fd9da81a8a4356ecb1b9d0341ed1, disabling compactions & flushes 2024-12-08T01:18:38,457 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. 2024-12-08T01:18:38,457 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. 2024-12-08T01:18:38,457 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. after waiting 0 ms 2024-12-08T01:18:38,457 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. 2024-12-08T01:18:38,458 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. 2024-12-08T01:18:38,458 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 33e0fd9da81a8a4356ecb1b9d0341ed1: Waiting for close lock at 1733620718457Disabling compacts and flushes for region at 1733620718457Disabling writes for close at 1733620718457Writing region close event to WAL at 1733620718458 (+1 ms)Closed at 1733620718458 2024-12-08T01:18:38,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741864_1041 (size=93) 2024-12-08T01:18:38,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741864_1041 (size=93) 2024-12-08T01:18:38,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741864_1041 (size=93) 2024-12-08T01:18:38,465 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:38,465 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80557104:(num 1733620718410) 2024-12-08T01:18:38,466 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:38,468 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:38,480 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469, exclude list is [], retry=0 2024-12-08T01:18:38,483 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:38,483 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:38,484 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:38,485 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469 2024-12-08T01:18:38,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:38,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741866_1043 (size=344306) 2024-12-08T01:18:38,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741866_1043 (size=344306) 2024-12-08T01:18:38,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741866_1043 (size=344306) 2024-12-08T01:18:38,635 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469, size=336.2 K (344306bytes) 2024-12-08T01:18:38,635 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469 2024-12-08T01:18:38,636 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469 after 1ms 2024-12-08T01:18:38,640 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469 took 4ms 2024-12-08T01:18:38,643 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733620718469.temp 2024-12-08T01:18:38,645 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000000001-wal.1733620718469.temp 2024-12-08T01:18:38,674 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469 so closing down 2024-12-08T01:18:38,674 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:38,675 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:38,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741867_1044 (size=344306) 2024-12-08T01:18:38,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741867_1044 (size=344306) 2024-12-08T01:18:38,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741867_1044 (size=344306) 2024-12-08T01:18:38,678 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000000001-wal.1733620718469.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-12-08T01:18:38,679 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000000001-wal.1733620718469.temp to hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003000 2024-12-08T01:18:38,680 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-12-08T01:18:38,680 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469, journal: Splitting hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469, size=336.2 K (344306bytes) at 1733620718635Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000000001-wal.1733620718469.temp at 1733620718645 (+10 ms)Split 1024 edits, skipped 0 edits. at 1733620718652 (+7 ms)Split 2048 edits, skipped 0 edits. at 1733620718665 (+13 ms)Finishing writing output for hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469 so closing down at 1733620718674 (+9 ms)3 split writer threads finished at 1733620718675 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000000001-wal.1733620718469.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1733620718678 (+3 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000000001-wal.1733620718469.temp to hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003000 at 1733620718679 (+1 ms)Processed 3000 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1733620718680 (+1 ms) 2024-12-08T01:18:38,681 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718469 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620718469 2024-12-08T01:18:38,682 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003000 2024-12-08T01:18:38,683 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:38,685 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:38,698 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686, exclude list is [], retry=0 2024-12-08T01:18:38,701 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:38,702 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:38,702 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:38,705 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686 2024-12-08T01:18:38,705 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:38,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741868_1045 (size=344433) 2024-12-08T01:18:38,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741868_1045 (size=344433) 2024-12-08T01:18:38,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741868_1045 (size=344433) 2024-12-08T01:18:38,853 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686, size=336.4 K (344433bytes) 2024-12-08T01:18:38,854 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686 2024-12-08T01:18:38,854 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686 after 0ms 2024-12-08T01:18:38,858 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686 took 5ms 2024-12-08T01:18:38,862 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733620718686.temp 2024-12-08T01:18:38,863 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003001-wal.1733620718686.temp 2024-12-08T01:18:38,911 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686 so closing down 2024-12-08T01:18:38,911 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:38,913 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:38,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741869_1046 (size=344433) 2024-12-08T01:18:38,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741869_1046 (size=344433) 2024-12-08T01:18:38,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741869_1046 (size=344433) 2024-12-08T01:18:38,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003001-wal.1733620718686.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-12-08T01:18:38,923 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003001-wal.1733620718686.temp to hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000006000 2024-12-08T01:18:38,923 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-12-08T01:18:38,924 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686, journal: Splitting hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686, size=336.4 K (344433bytes) at 1733620718854Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003001-wal.1733620718686.temp at 1733620718863 (+9 ms)Split 1024 edits, skipped 0 edits. at 1733620718870 (+7 ms)Split 2048 edits, skipped 0 edits. at 1733620718877 (+7 ms)Finishing writing output for hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686 so closing down at 1733620718911 (+34 ms)3 split writer threads finished at 1733620718913 (+2 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003001-wal.1733620718686.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1733620718921 (+8 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003001-wal.1733620718686.temp to hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000006000 at 1733620718923 (+2 ms)Processed 3000 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1733620718923 2024-12-08T01:18:38,926 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718686 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620718686 2024-12-08T01:18:38,927 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000006000 2024-12-08T01:18:38,927 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:38,929 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1733620718402, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:38,948 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718930, exclude list is [], retry=0 2024-12-08T01:18:38,951 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:38,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:38,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:38,967 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733620718402/wal.1733620718930 2024-12-08T01:18:38,968 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:38,969 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 33e0fd9da81a8a4356ecb1b9d0341ed1, NAME => 'test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:38,969 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:38,969 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,969 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,973 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,974 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33e0fd9da81a8a4356ecb1b9d0341ed1 columnFamilyName a 2024-12-08T01:18:38,975 DEBUG [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:38,975 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(327): Store=33e0fd9da81a8a4356ecb1b9d0341ed1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:38,976 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,977 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33e0fd9da81a8a4356ecb1b9d0341ed1 columnFamilyName b 2024-12-08T01:18:38,977 DEBUG [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:38,977 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(327): Store=33e0fd9da81a8a4356ecb1b9d0341ed1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:38,978 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,979 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33e0fd9da81a8a4356ecb1b9d0341ed1 columnFamilyName c 2024-12-08T01:18:38,979 DEBUG [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:38,979 INFO [StoreOpener-33e0fd9da81a8a4356ecb1b9d0341ed1-1 {}] regionserver.HStore(327): Store=33e0fd9da81a8a4356ecb1b9d0341ed1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:38,980 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,980 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,983 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:38,983 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003000 2024-12-08T01:18:39,012 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003000 2024-12-08T01:18:39,013 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000006000 2024-12-08T01:18:39,045 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000006000 2024-12-08T01:18:39,045 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-08T01:18:39,074 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/a/beac8aef643140f185e4d880f1bb215c is 41, key is test2727/a:100/1733620718709/Put/seqid=0 2024-12-08T01:18:39,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741871_1048 (size=84227) 2024-12-08T01:18:39,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741871_1048 (size=84227) 2024-12-08T01:18:39,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741871_1048 (size=84227) 2024-12-08T01:18:39,080 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/a/beac8aef643140f185e4d880f1bb215c 2024-12-08T01:18:39,115 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/b/c85aeb27d7ba45beacce9c82a5c81bc3 is 41, key is test2727/b:100/1733620718749/Put/seqid=0 2024-12-08T01:18:39,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741872_1049 (size=84609) 2024-12-08T01:18:39,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741872_1049 (size=84609) 2024-12-08T01:18:39,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741872_1049 (size=84609) 2024-12-08T01:18:39,122 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/b/c85aeb27d7ba45beacce9c82a5c81bc3 2024-12-08T01:18:39,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/c/d570b73987004dd59893d4bc93548bc0 is 41, key is test2727/c:100/1733620718789/Put/seqid=0 2024-12-08T01:18:39,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741873_1050 (size=84609) 2024-12-08T01:18:39,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741873_1050 (size=84609) 2024-12-08T01:18:39,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741873_1050 (size=84609) 2024-12-08T01:18:39,154 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/c/d570b73987004dd59893d4bc93548bc0 2024-12-08T01:18:39,162 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/a/beac8aef643140f185e4d880f1bb215c as hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/a/beac8aef643140f185e4d880f1bb215c 2024-12-08T01:18:39,170 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/a/beac8aef643140f185e4d880f1bb215c, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-08T01:18:39,171 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/b/c85aeb27d7ba45beacce9c82a5c81bc3 as hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/b/c85aeb27d7ba45beacce9c82a5c81bc3 2024-12-08T01:18:39,178 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/b/c85aeb27d7ba45beacce9c82a5c81bc3, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-08T01:18:39,179 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/.tmp/c/d570b73987004dd59893d4bc93548bc0 as hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/c/d570b73987004dd59893d4bc93548bc0 2024-12-08T01:18:39,186 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/c/d570b73987004dd59893d4bc93548bc0, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-08T01:18:39,186 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 33e0fd9da81a8a4356ecb1b9d0341ed1 in 141ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-08T01:18:39,187 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000003000 2024-12-08T01:18:39,188 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/0000000000000006000 2024-12-08T01:18:39,189 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:39,189 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:39,190 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:39,191 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33e0fd9da81a8a4356ecb1b9d0341ed1 2024-12-08T01:18:39,194 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/test2727/33e0fd9da81a8a4356ecb1b9d0341ed1/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-08T01:18:39,195 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33e0fd9da81a8a4356ecb1b9d0341ed1; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62223262, jitterRate=-0.0728011429309845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:39,196 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33e0fd9da81a8a4356ecb1b9d0341ed1: Writing region info on filesystem at 1733620718969Initializing all the Stores at 1733620718970 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620718970Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620718972 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620718973 (+1 ms)Obtaining lock to block concurrent updates at 1733620719045 (+72 ms)Preparing flush snapshotting stores in 33e0fd9da81a8a4356ecb1b9d0341ed1 at 1733620719045Finished memstore snapshotting test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733620719046 (+1 ms)Flushing stores of test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. at 1733620719046Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/a: creating writer at 1733620719046Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/a: appending metadata at 1733620719072 (+26 ms)Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/a: closing flushed file at 1733620719073 (+1 ms)Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/b: creating writer at 1733620719088 (+15 ms)Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/b: appending metadata at 1733620719113 (+25 ms)Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/b: closing flushed file at 1733620719113Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/c: creating writer at 1733620719129 (+16 ms)Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/c: appending metadata at 1733620719146 (+17 ms)Flushing 33e0fd9da81a8a4356ecb1b9d0341ed1/c: closing flushed file at 1733620719146Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@350c3e4c: reopening flushed file at 1733620719161 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e609932: reopening flushed file at 1733620719170 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@374d734c: reopening flushed file at 1733620719178 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 33e0fd9da81a8a4356ecb1b9d0341ed1 in 141ms, sequenceid=6000, compaction requested=false; wal=null at 1733620719187 (+9 ms)Cleaning up temporary data from old regions at 1733620719189 (+2 ms)Region opened successfully at 1733620719196 (+7 ms) 2024-12-08T01:18:39,197 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-08T01:18:39,197 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 33e0fd9da81a8a4356ecb1b9d0341ed1, disabling compactions & flushes 2024-12-08T01:18:39,197 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. 2024-12-08T01:18:39,197 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. 2024-12-08T01:18:39,197 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. after waiting 0 ms 2024-12-08T01:18:39,197 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. 2024-12-08T01:18:39,198 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733620718404.33e0fd9da81a8a4356ecb1b9d0341ed1. 2024-12-08T01:18:39,198 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 33e0fd9da81a8a4356ecb1b9d0341ed1: Waiting for close lock at 1733620719197Disabling compacts and flushes for region at 1733620719197Disabling writes for close at 1733620719197Writing region close event to WAL at 1733620719198 (+1 ms)Closed at 1733620719198 2024-12-08T01:18:39,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741870_1047 (size=93) 2024-12-08T01:18:39,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741870_1047 (size=93) 2024-12-08T01:18:39,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741870_1047 (size=93) 2024-12-08T01:18:39,203 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:39,203 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620718930) 2024-12-08T01:18:39,214 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404 (was 402) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37882 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37380 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:44114 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=267 (was 267), ProcessCount=11 (was 11), AvailableMemoryMB=10285 (was 10471) 2024-12-08T01:18:39,224 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=267, ProcessCount=11, AvailableMemoryMB=10285 2024-12-08T01:18:39,237 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:39,243 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:39,243 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733620719243 2024-12-08T01:18:39,251 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243 2024-12-08T01:18:39,252 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:39,254 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 93b043e2cae94d49b3fb752e1b62bb1f, NAME => 'testSequentialEditLogSeqNum,,1733620719238.93b043e2cae94d49b3fb752e1b62bb1f.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:39,254 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733620719238.93b043e2cae94d49b3fb752e1b62bb1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:39,254 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,254 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,255 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f doesn't exist for region: 93b043e2cae94d49b3fb752e1b62bb1f on table testSequentialEditLogSeqNum 2024-12-08T01:18:39,255 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 93b043e2cae94d49b3fb752e1b62bb1f on table testSequentialEditLogSeqNum 2024-12-08T01:18:39,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741875_1052 (size=62) 2024-12-08T01:18:39,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741875_1052 (size=62) 2024-12-08T01:18:39,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741875_1052 (size=62) 2024-12-08T01:18:39,267 INFO [StoreOpener-93b043e2cae94d49b3fb752e1b62bb1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,269 INFO [StoreOpener-93b043e2cae94d49b3fb752e1b62bb1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93b043e2cae94d49b3fb752e1b62bb1f columnFamilyName a 2024-12-08T01:18:39,269 DEBUG [StoreOpener-93b043e2cae94d49b3fb752e1b62bb1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:39,270 INFO [StoreOpener-93b043e2cae94d49b3fb752e1b62bb1f-1 {}] regionserver.HStore(327): Store=93b043e2cae94d49b3fb752e1b62bb1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:39,270 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,271 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,271 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,272 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,272 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,274 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 93b043e2cae94d49b3fb752e1b62bb1f 2024-12-08T01:18:39,276 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:39,277 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 93b043e2cae94d49b3fb752e1b62bb1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62101560, jitterRate=-0.07461464405059814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:39,278 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 93b043e2cae94d49b3fb752e1b62bb1f: Writing region info on filesystem at 1733620719254Initializing all the Stores at 1733620719267 (+13 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620719267Cleaning up temporary data from old regions at 1733620719272 (+5 ms)Region opened successfully at 1733620719277 (+5 ms) 2024-12-08T01:18:39,288 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 93b043e2cae94d49b3fb752e1b62bb1f 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-08T01:18:39,312 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/.tmp/a/245df86d432d43fbb22ce2518928bb2e is 81, key is testSequentialEditLogSeqNum/a:x0/1733620719278/Put/seqid=0 2024-12-08T01:18:39,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741876_1053 (size=5833) 2024-12-08T01:18:39,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741876_1053 (size=5833) 2024-12-08T01:18:39,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741876_1053 (size=5833) 2024-12-08T01:18:39,320 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/.tmp/a/245df86d432d43fbb22ce2518928bb2e 2024-12-08T01:18:39,328 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/.tmp/a/245df86d432d43fbb22ce2518928bb2e as hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/a/245df86d432d43fbb22ce2518928bb2e 2024-12-08T01:18:39,335 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/a/245df86d432d43fbb22ce2518928bb2e, entries=10, sequenceid=13, filesize=5.7 K 2024-12-08T01:18:39,337 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 93b043e2cae94d49b3fb752e1b62bb1f in 49ms, sequenceid=13, compaction requested=false 2024-12-08T01:18:39,337 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 93b043e2cae94d49b3fb752e1b62bb1f: 2024-12-08T01:18:39,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:18:39,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:18:39,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:18:39,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:18:39,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:18:39,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741874_1051 (size=3017) 2024-12-08T01:18:39,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741874_1051 (size=3017) 2024-12-08T01:18:39,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741874_1051 (size=3017) 2024-12-08T01:18:39,361 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243, size=2.9 K (3017bytes) 2024-12-08T01:18:39,361 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243 2024-12-08T01:18:39,362 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243 after 1ms 2024-12-08T01:18:39,365 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243 took 4ms 2024-12-08T01:18:39,367 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243 so closing down 2024-12-08T01:18:39,367 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:39,369 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620719243.temp 2024-12-08T01:18:39,370 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/0000000000000000003-wal.1733620719243.temp 2024-12-08T01:18:39,371 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:39,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741877_1054 (size=2433) 2024-12-08T01:18:39,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741877_1054 (size=2433) 2024-12-08T01:18:39,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741877_1054 (size=2433) 2024-12-08T01:18:39,378 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/0000000000000000003-wal.1733620719243.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:39,380 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/0000000000000000003-wal.1733620719243.temp to hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/0000000000000000020 2024-12-08T01:18:39,380 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-12-08T01:18:39,380 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243, journal: Splitting hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243, size=2.9 K (3017bytes) at 1733620719361Finishing writing output for hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243 so closing down at 1733620719367 (+6 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/0000000000000000003-wal.1733620719243.temp at 1733620719370 (+3 ms)3 split writer threads finished at 1733620719371 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/0000000000000000003-wal.1733620719243.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733620719378 (+7 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/0000000000000000003-wal.1733620719243.temp to hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/93b043e2cae94d49b3fb752e1b62bb1f/recovered.edits/0000000000000000020 at 1733620719380 (+2 ms)Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620719236/wal.1733620719243, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1733620719380 2024-12-08T01:18:39,401 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37882 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:44114 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 267) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10268 (was 10285) 2024-12-08T01:18:39,415 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=10264 2024-12-08T01:18:39,430 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:39,432 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:39,465 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:39,469 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97521382, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-97521382, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:39,483 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97521382/hregion-97521382.1733620719469, exclude list is [], retry=0 2024-12-08T01:18:39,486 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:39,486 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:39,486 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:39,489 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97521382/hregion-97521382.1733620719469 2024-12-08T01:18:39,489 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:39,489 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 496d0a059050a1ebec2b42b8a7ce127c, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:39,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741879_1056 (size=70) 2024-12-08T01:18:39,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741879_1056 (size=70) 2024-12-08T01:18:39,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741879_1056 (size=70) 2024-12-08T01:18:39,503 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:39,504 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,507 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName a 2024-12-08T01:18:39,507 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:39,508 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:39,508 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,509 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName b 2024-12-08T01:18:39,510 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:39,510 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:39,510 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,512 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName c 2024-12-08T01:18:39,512 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:39,513 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:39,513 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,514 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,514 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,515 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,515 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,516 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:39,518 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,521 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:39,522 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 496d0a059050a1ebec2b42b8a7ce127c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71765429, jitterRate=0.06938822567462921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:39,523 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 496d0a059050a1ebec2b42b8a7ce127c: Writing region info on filesystem at 1733620719503Initializing all the Stores at 1733620719504 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620719504Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620719504Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620719504Cleaning up temporary data from old regions at 1733620719515 (+11 ms)Region opened successfully at 1733620719522 (+7 ms) 2024-12-08T01:18:39,523 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 496d0a059050a1ebec2b42b8a7ce127c, disabling compactions & flushes 2024-12-08T01:18:39,523 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. 2024-12-08T01:18:39,523 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. 2024-12-08T01:18:39,523 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. after waiting 0 ms 2024-12-08T01:18:39,523 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. 2024-12-08T01:18:39,524 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. 2024-12-08T01:18:39,524 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 496d0a059050a1ebec2b42b8a7ce127c: Waiting for close lock at 1733620719523Disabling compacts and flushes for region at 1733620719523Disabling writes for close at 1733620719523Writing region close event to WAL at 1733620719524 (+1 ms)Closed at 1733620719524 2024-12-08T01:18:39,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741878_1055 (size=93) 2024-12-08T01:18:39,529 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-97521382/hregion-97521382.1733620719469 not finished, retry = 0 2024-12-08T01:18:39,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741878_1055 (size=93) 2024-12-08T01:18:39,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741878_1055 (size=93) 2024-12-08T01:18:39,553 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T01:18:39,553 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:39,554 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T01:18:39,554 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T01:18:39,555 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T01:18:39,555 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:39,555 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-08T01:18:39,555 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:39,556 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-08T01:18:39,556 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:39,632 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:39,632 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97521382:(num 1733620719469) 2024-12-08T01:18:39,632 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:39,635 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:39,649 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636, exclude list is [], retry=0 2024-12-08T01:18:39,653 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:39,653 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:39,653 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:39,655 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 2024-12-08T01:18:39,656 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:39,656 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 496d0a059050a1ebec2b42b8a7ce127c, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:39,656 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:39,656 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,656 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,657 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,658 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName a 2024-12-08T01:18:39,658 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:39,659 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:39,659 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,660 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName b 2024-12-08T01:18:39,660 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:39,661 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:39,661 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,662 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName c 2024-12-08T01:18:39,662 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:39,662 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:39,662 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,664 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,665 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,665 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,666 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:39,667 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:39,668 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 496d0a059050a1ebec2b42b8a7ce127c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73252671, jitterRate=0.09154985845088959}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:39,669 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 496d0a059050a1ebec2b42b8a7ce127c: Writing region info on filesystem at 1733620719656Initializing all the Stores at 1733620719657 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620719657Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620719657Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620719657Cleaning up temporary data from old regions at 1733620719665 (+8 ms)Region opened successfully at 1733620719669 (+4 ms) 2024-12-08T01:18:39,673 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733620719672/Put/seqid=0 2024-12-08T01:18:39,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741881_1058 (size=4826) 2024-12-08T01:18:39,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741881_1058 (size=4826) 2024-12-08T01:18:39,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741881_1058 (size=4826) 2024-12-08T01:18:39,682 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 496d0a059050a1ebec2b42b8a7ce127c/a 2024-12-08T01:18:39,690 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-08T01:18:39,691 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-08T01:18:39,691 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 496d0a059050a1ebec2b42b8a7ce127c: 2024-12-08T01:18:39,692 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/a/c81b4fd41336430d8fa79b518139eb71_SeqId_3_ 2024-12-08T01:18:39,693 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 496d0a059050a1ebec2b42b8a7ce127c/a as hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/a/c81b4fd41336430d8fa79b518139eb71_SeqId_3_ - updating store file list. 2024-12-08T01:18:39,698 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c81b4fd41336430d8fa79b518139eb71_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-08T01:18:39,699 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/a/c81b4fd41336430d8fa79b518139eb71_SeqId_3_ into 496d0a059050a1ebec2b42b8a7ce127c/a 2024-12-08T01:18:39,699 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 496d0a059050a1ebec2b42b8a7ce127c/a (new location: hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/a/c81b4fd41336430d8fa79b518139eb71_SeqId_3_) 2024-12-08T01:18:39,734 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636, size=0 (0bytes) 2024-12-08T01:18:39,734 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 might be still open, length is 0 2024-12-08T01:18:39,734 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 2024-12-08T01:18:39,735 WARN [IPC Server handler 4 on default port 46839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-08T01:18:39,735 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 after 1ms 2024-12-08T01:18:40,329 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:44238 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39423:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44238 dst: /127.0.0.1:39423 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39423 remote=/127.0.0.1:44238]. Total timeout mills is 60000, 59375 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:40,330 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37978 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46125:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37978 dst: /127.0.0.1:46125 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:40,330 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37474 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37474 dst: /127.0.0.1:45653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:40,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741880_1059 (size=504) 2024-12-08T01:18:40,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741880_1059 (size=504) 2024-12-08T01:18:40,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741880_1059 (size=504) 2024-12-08T01:18:43,737 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 after 4003ms 2024-12-08T01:18:43,745 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 took 4011ms 2024-12-08T01:18:43,748 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636; continuing. 2024-12-08T01:18:43,748 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 so closing down 2024-12-08T01:18:43,748 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:43,750 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733620719636.temp 2024-12-08T01:18:43,752 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005-wal.1733620719636.temp 2024-12-08T01:18:43,753 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:43,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741882_1060 (size=238) 2024-12-08T01:18:43,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741882_1060 (size=238) 2024-12-08T01:18:43,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741882_1060 (size=238) 2024-12-08T01:18:43,764 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005-wal.1733620719636.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:43,766 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005-wal.1733620719636.temp to hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005 2024-12-08T01:18:43,766 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 20 ms; skipped=1; WAL=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636, size=0, length=0, corrupted=false, cancelled=false 2024-12-08T01:18:43,766 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636, journal: Splitting hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636, size=0 (0bytes) at 1733620719734Finishing writing output for hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 so closing down at 1733620723748 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005-wal.1733620719636.temp at 1733620723752 (+4 ms)3 split writer threads finished at 1733620723753 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005-wal.1733620719636.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733620723764 (+11 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005-wal.1733620719636.temp to hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005 at 1733620723766 (+2 ms)Processed 2 edits across 1 Regions in 20 ms; skipped=1; WAL=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636, size=0, length=0, corrupted=false, cancelled=false at 1733620723766 2024-12-08T01:18:43,769 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620719636 2024-12-08T01:18:43,770 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005 2024-12-08T01:18:43,770 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:43,772 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:43,789 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620723773, exclude list is [], retry=0 2024-12-08T01:18:43,792 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:43,792 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:43,793 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:43,794 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620723773 2024-12-08T01:18:43,795 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:18:43,795 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 496d0a059050a1ebec2b42b8a7ce127c, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:43,795 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:43,795 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,795 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,797 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,798 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName a 2024-12-08T01:18:43,798 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:43,804 DEBUG [StoreFileOpener-496d0a059050a1ebec2b42b8a7ce127c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c81b4fd41336430d8fa79b518139eb71_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-08T01:18:43,805 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/a/c81b4fd41336430d8fa79b518139eb71_SeqId_3_ 2024-12-08T01:18:43,805 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:43,805 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,806 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName b 2024-12-08T01:18:43,806 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:43,806 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:43,807 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,807 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496d0a059050a1ebec2b42b8a7ce127c columnFamilyName c 2024-12-08T01:18:43,807 DEBUG [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:43,808 INFO [StoreOpener-496d0a059050a1ebec2b42b8a7ce127c-1 {}] regionserver.HStore(327): Store=496d0a059050a1ebec2b42b8a7ce127c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,809 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,810 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,811 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005 2024-12-08T01:18:43,814 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005 2024-12-08T01:18:43,814 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 496d0a059050a1ebec2b42b8a7ce127c 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-08T01:18:43,827 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/.tmp/a/048d2817422b43cdb7defe87f79e95d4 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733620719703/Put/seqid=0 2024-12-08T01:18:43,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741884_1062 (size=5149) 2024-12-08T01:18:43,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741884_1062 (size=5149) 2024-12-08T01:18:43,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741884_1062 (size=5149) 2024-12-08T01:18:43,834 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/.tmp/a/048d2817422b43cdb7defe87f79e95d4 2024-12-08T01:18:43,840 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/.tmp/a/048d2817422b43cdb7defe87f79e95d4 as hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/a/048d2817422b43cdb7defe87f79e95d4 2024-12-08T01:18:43,846 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/a/048d2817422b43cdb7defe87f79e95d4, entries=1, sequenceid=5, filesize=5.0 K 2024-12-08T01:18:43,846 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 496d0a059050a1ebec2b42b8a7ce127c in 32ms, sequenceid=5, compaction requested=false; wal=null 2024-12-08T01:18:43,847 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/0000000000000000005 2024-12-08T01:18:43,848 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,848 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,848 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:43,850 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 496d0a059050a1ebec2b42b8a7ce127c 2024-12-08T01:18:43,852 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/496d0a059050a1ebec2b42b8a7ce127c/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-08T01:18:43,853 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 496d0a059050a1ebec2b42b8a7ce127c; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72835195, jitterRate=0.08532898128032684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:43,853 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 496d0a059050a1ebec2b42b8a7ce127c: Writing region info on filesystem at 1733620723795Initializing all the Stores at 1733620723796 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620723796Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620723797 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620723797Obtaining lock to block concurrent updates at 1733620723814 (+17 ms)Preparing flush snapshotting stores in 496d0a059050a1ebec2b42b8a7ce127c at 1733620723814Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733620723814Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. at 1733620723814Flushing 496d0a059050a1ebec2b42b8a7ce127c/a: creating writer at 1733620723814Flushing 496d0a059050a1ebec2b42b8a7ce127c/a: appending metadata at 1733620723826 (+12 ms)Flushing 496d0a059050a1ebec2b42b8a7ce127c/a: closing flushed file at 1733620723826Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4de74c11: reopening flushed file at 1733620723839 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 496d0a059050a1ebec2b42b8a7ce127c in 32ms, sequenceid=5, compaction requested=false; wal=null at 1733620723846 (+7 ms)Cleaning up temporary data from old regions at 1733620723848 (+2 ms)Region opened successfully at 1733620723853 (+5 ms) 2024-12-08T01:18:43,856 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 496d0a059050a1ebec2b42b8a7ce127c, disabling compactions & flushes 2024-12-08T01:18:43,856 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. 2024-12-08T01:18:43,856 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. 2024-12-08T01:18:43,856 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. after waiting 0 ms 2024-12-08T01:18:43,856 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. 2024-12-08T01:18:43,857 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733620719430.496d0a059050a1ebec2b42b8a7ce127c. 2024-12-08T01:18:43,857 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 496d0a059050a1ebec2b42b8a7ce127c: Waiting for close lock at 1733620723856Disabling compacts and flushes for region at 1733620723856Disabling writes for close at 1733620723856Writing region close event to WAL at 1733620723857 (+1 ms)Closed at 1733620723857 2024-12-08T01:18:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741883_1061 (size=93) 2024-12-08T01:18:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741883_1061 (size=93) 2024-12-08T01:18:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741883_1061 (size=93) 2024-12-08T01:18:43,862 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:43,862 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620723773) 2024-12-08T01:18:43,875 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 409) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:46839 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-182014877_22 at /127.0.0.1:42586 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-182014877_22 at /127.0.0.1:38032 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:46839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1021 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=10256 (was 10264) 2024-12-08T01:18:43,887 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1021, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=10255 2024-12-08T01:18:43,903 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:43,907 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:18:43,911 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,33573,1733620709230 2024-12-08T01:18:43,913 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@547ede2f 2024-12-08T01:18:43,914 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:18:43,916 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:18:43,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:18:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-08T01:18:43,931 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T01:18:43,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-08T01:18:43,933 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:43,935 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T01:18:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:18:43,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741885_1063 (size=694) 2024-12-08T01:18:43,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741885_1063 (size=694) 2024-12-08T01:18:43,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741885_1063 (size=694) 2024-12-08T01:18:43,948 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f837f401ea75f84ac401a4defcd14e10, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc 2024-12-08T01:18:43,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741886_1064 (size=77) 2024-12-08T01:18:43,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741886_1064 (size=77) 2024-12-08T01:18:43,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741886_1064 (size=77) 2024-12-08T01:18:43,958 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:43,958 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing f837f401ea75f84ac401a4defcd14e10, disabling compactions & flushes 2024-12-08T01:18:43,958 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:43,958 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:43,958 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. after waiting 0 ms 2024-12-08T01:18:43,958 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:43,958 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:43,958 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for f837f401ea75f84ac401a4defcd14e10: Waiting for close lock at 1733620723958Disabling compacts and flushes for region at 1733620723958Disabling writes for close at 1733620723958Writing region close event to WAL at 1733620723958Closed at 1733620723958 2024-12-08T01:18:43,960 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T01:18:43,965 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733620723960"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733620723960"}]},"ts":"1733620723960"} 2024-12-08T01:18:43,973 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T01:18:43,975 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T01:18:43,977 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620723975"}]},"ts":"1733620723975"} 2024-12-08T01:18:43,982 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-08T01:18:43,982 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {96982154c400=0} racks are {/default-rack=0} 2024-12-08T01:18:43,983 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T01:18:43,983 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T01:18:43,983 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-08T01:18:43,984 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T01:18:43,984 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T01:18:43,984 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-08T01:18:43,984 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T01:18:43,984 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T01:18:43,984 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-08T01:18:43,984 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T01:18:43,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN}] 2024-12-08T01:18:43,987 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN 2024-12-08T01:18:43,988 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN; state=OFFLINE, location=96982154c400,35607,1733620710216; forceNewPlan=false, retain=false 2024-12-08T01:18:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:18:44,142 INFO [96982154c400:33573 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T01:18:44,143 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=OPENING, regionLocation=96982154c400,35607,1733620710216 2024-12-08T01:18:44,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN because future has completed 2024-12-08T01:18:44,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216}] 2024-12-08T01:18:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:18:44,312 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,313 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f837f401ea75f84ac401a4defcd14e10, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:44,313 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,314 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:44,314 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,314 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,317 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,320 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf1 2024-12-08T01:18:44,320 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:44,321 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:44,321 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,323 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf2 2024-12-08T01:18:44,323 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:44,324 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:44,324 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,325 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,326 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,327 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,327 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,329 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-08T01:18:44,331 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,334 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:44,335 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f837f401ea75f84ac401a4defcd14e10; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61956401, jitterRate=-0.07677768170833588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-08T01:18:44,335 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,337 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f837f401ea75f84ac401a4defcd14e10: Running coprocessor pre-open hook at 1733620724314Writing region info on filesystem at 1733620724314Initializing all the Stores at 1733620724316 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620724316Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620724317 (+1 ms)Cleaning up temporary data from old regions at 1733620724327 (+10 ms)Running coprocessor post-open hooks at 1733620724335 (+8 ms)Region opened successfully at 1733620724337 (+2 ms) 2024-12-08T01:18:44,339 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., pid=6, masterSystemTime=1733620724303 2024-12-08T01:18:44,342 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,342 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,343 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,35607,1733620710216 2024-12-08T01:18:44,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216 because future has completed 2024-12-08T01:18:44,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T01:18:44,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216 in 198 msec 2024-12-08T01:18:44,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T01:18:44,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN in 364 msec 2024-12-08T01:18:44,353 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T01:18:44,353 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620724353"}]},"ts":"1733620724353"} 2024-12-08T01:18:44,355 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-08T01:18:44,357 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T01:18:44,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 434 msec 2024-12-08T01:18:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:18:44,565 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-08T01:18:44,565 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-08T01:18:44,566 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T01:18:44,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-08T01:18:44,573 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T01:18:44,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-08T01:18:44,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=2] 2024-12-08T01:18:44,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f837f401ea75f84ac401a4defcd14e10, source=96982154c400,35607,1733620710216, destination=96982154c400,41905,1733620710058, warming up region on 96982154c400,41905,1733620710058 2024-12-08T01:18:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:18:44,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f837f401ea75f84ac401a4defcd14e10, source=96982154c400,35607,1733620710216, destination=96982154c400,41905,1733620710058, running balancer 2024-12-08T01:18:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE 2024-12-08T01:18:44,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54679, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:18:44,609 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE 2024-12-08T01:18:44,611 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=CLOSING, regionLocation=96982154c400,35607,1733620710216 2024-12-08T01:18:44,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(7855): Warmup {ENCODED => f837f401ea75f84ac401a4defcd14e10, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:44,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE because future has completed 2024-12-08T01:18:44,615 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,615 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:18:44,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216}] 2024-12-08T01:18:44,616 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf1 2024-12-08T01:18:44,616 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:44,616 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:44,617 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,618 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf2 2024-12-08T01:18:44,618 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:44,618 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(1722): Closing f837f401ea75f84ac401a4defcd14e10, disabling compactions & flushes 2024-12-08T01:18:44,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. after waiting 0 ms 2024-12-08T01:18:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] regionserver.HRegion(1676): Region close journal for f837f401ea75f84ac401a4defcd14e10: Waiting for close lock at 1733620724618Disabling compacts and flushes for region at 1733620724618Disabling writes for close at 1733620724619 (+1 ms)Writing region close event to WAL at 1733620724619Closed at 1733620724619 2024-12-08T01:18:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-08T01:18:44,775 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,776 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:18:44,776 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing f837f401ea75f84ac401a4defcd14e10, disabling compactions & flushes 2024-12-08T01:18:44,777 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,777 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,777 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. after waiting 0 ms 2024-12-08T01:18:44,777 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,777 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing f837f401ea75f84ac401a4defcd14e10 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-08T01:18:44,799 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf1/baf541d2fd86407db163d57684d812a1 is 35, key is r1/cf1:q/1733620724592/Put/seqid=0 2024-12-08T01:18:44,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741887_1065 (size=4783) 2024-12-08T01:18:44,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741887_1065 (size=4783) 2024-12-08T01:18:44,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741887_1065 (size=4783) 2024-12-08T01:18:44,807 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf1/baf541d2fd86407db163d57684d812a1 2024-12-08T01:18:44,815 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf1/baf541d2fd86407db163d57684d812a1 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/baf541d2fd86407db163d57684d812a1 2024-12-08T01:18:44,822 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/baf541d2fd86407db163d57684d812a1, entries=1, sequenceid=5, filesize=4.7 K 2024-12-08T01:18:44,823 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for f837f401ea75f84ac401a4defcd14e10 in 46ms, sequenceid=5, compaction requested=false 2024-12-08T01:18:44,823 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-08T01:18:44,828 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-08T01:18:44,831 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:44,831 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for f837f401ea75f84ac401a4defcd14e10: Waiting for close lock at 1733620724776Running coprocessor pre-close hooks at 1733620724776Disabling compacts and flushes for region at 1733620724776Disabling writes for close at 1733620724777 (+1 ms)Obtaining lock to block concurrent updates at 1733620724777Preparing flush snapshotting stores in f837f401ea75f84ac401a4defcd14e10 at 1733620724777Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733620724777Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. at 1733620724779 (+2 ms)Flushing f837f401ea75f84ac401a4defcd14e10/cf1: creating writer at 1733620724779Flushing f837f401ea75f84ac401a4defcd14e10/cf1: appending metadata at 1733620724799 (+20 ms)Flushing f837f401ea75f84ac401a4defcd14e10/cf1: closing flushed file at 1733620724799Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27b4d904: reopening flushed file at 1733620724814 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for f837f401ea75f84ac401a4defcd14e10 in 46ms, sequenceid=5, compaction requested=false at 1733620724823 (+9 ms)Writing region close event to WAL at 1733620724824 (+1 ms)Running coprocessor post-close hooks at 1733620724829 (+5 ms)Closed at 1733620724831 (+2 ms) 2024-12-08T01:18:44,831 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding f837f401ea75f84ac401a4defcd14e10 move to 96982154c400,41905,1733620710058 record at close sequenceid=5 2024-12-08T01:18:44,834 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:44,835 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=CLOSED 2024-12-08T01:18:44,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216 because future has completed 2024-12-08T01:18:44,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T01:18:44,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216 in 225 msec 2024-12-08T01:18:44,845 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE; state=CLOSED, location=96982154c400,41905,1733620710058; forceNewPlan=false, retain=false 2024-12-08T01:18:44,996 INFO [96982154c400:33573 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T01:18:44,997 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=OPENING, regionLocation=96982154c400,41905,1733620710058 2024-12-08T01:18:45,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE because future has completed 2024-12-08T01:18:45,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058}] 2024-12-08T01:18:45,161 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,161 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f837f401ea75f84ac401a4defcd14e10, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:45,161 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,161 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:45,162 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,162 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,163 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,164 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf1 2024-12-08T01:18:45,164 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:45,172 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/baf541d2fd86407db163d57684d812a1 2024-12-08T01:18:45,172 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:45,172 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,173 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf2 2024-12-08T01:18:45,173 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:45,174 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:45,174 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,175 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,177 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,177 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,178 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,178 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-08T01:18:45,180 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,181 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f837f401ea75f84ac401a4defcd14e10; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62300356, jitterRate=-0.071652352809906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-08T01:18:45,181 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,182 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f837f401ea75f84ac401a4defcd14e10: Running coprocessor pre-open hook at 1733620725162Writing region info on filesystem at 1733620725162Initializing all the Stores at 1733620725163 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620725163Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620725163Cleaning up temporary data from old regions at 1733620725178 (+15 ms)Running coprocessor post-open hooks at 1733620725182 (+4 ms)Region opened successfully at 1733620725182 2024-12-08T01:18:45,184 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., pid=9, masterSystemTime=1733620725157 2024-12-08T01:18:45,187 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,187 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,188 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=OPEN, openSeqNum=9, regionLocation=96982154c400,41905,1733620710058 2024-12-08T01:18:45,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058 because future has completed 2024-12-08T01:18:45,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-08T01:18:45,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058 in 189 msec 2024-12-08T01:18:45,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE in 589 msec 2024-12-08T01:18:45,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:18:45,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:18:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:33990 deadline: 1733620785220, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=41905 startCode=1733620710058. As of locationSeqNum=5. 2024-12-08T01:18:45,242 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=41905 startCode=1733620710058. As of locationSeqNum=5. 2024-12-08T01:18:45,242 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=41905 startCode=1733620710058. As of locationSeqNum=5. 2024-12-08T01:18:45,242 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,41905,1733620710058, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=41905 startCode=1733620710058. As of locationSeqNum=5. 2024-12-08T01:18:45,354 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:18:45,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:18:45,366 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f837f401ea75f84ac401a4defcd14e10 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-08T01:18:45,380 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf1/d3f409a825864a5f96325b4bcd5f9528 is 29, key is r1/cf1:/1733620725357/DeleteFamily/seqid=0 2024-12-08T01:18:45,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741888_1066 (size=4906) 2024-12-08T01:18:45,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741888_1066 (size=4906) 2024-12-08T01:18:45,392 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf1/d3f409a825864a5f96325b4bcd5f9528 2024-12-08T01:18:45,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741888_1066 (size=4906) 2024-12-08T01:18:45,398 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d3f409a825864a5f96325b4bcd5f9528 2024-12-08T01:18:45,410 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf2/e0cd84d480c040428edb8802702b231e is 29, key is r1/cf2:/1733620725357/DeleteFamily/seqid=0 2024-12-08T01:18:45,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741889_1067 (size=4906) 2024-12-08T01:18:45,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741889_1067 (size=4906) 2024-12-08T01:18:45,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741889_1067 (size=4906) 2024-12-08T01:18:45,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf2/e0cd84d480c040428edb8802702b231e 2024-12-08T01:18:45,423 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0cd84d480c040428edb8802702b231e 2024-12-08T01:18:45,424 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf1/d3f409a825864a5f96325b4bcd5f9528 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/d3f409a825864a5f96325b4bcd5f9528 2024-12-08T01:18:45,430 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d3f409a825864a5f96325b4bcd5f9528 2024-12-08T01:18:45,431 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/d3f409a825864a5f96325b4bcd5f9528, entries=1, sequenceid=12, filesize=4.8 K 2024-12-08T01:18:45,432 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf2/e0cd84d480c040428edb8802702b231e as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/e0cd84d480c040428edb8802702b231e 2024-12-08T01:18:45,437 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0cd84d480c040428edb8802702b231e 2024-12-08T01:18:45,437 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/e0cd84d480c040428edb8802702b231e, entries=1, sequenceid=12, filesize=4.8 K 2024-12-08T01:18:45,438 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for f837f401ea75f84ac401a4defcd14e10 in 72ms, sequenceid=12, compaction requested=false 2024-12-08T01:18:45,439 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f837f401ea75f84ac401a4defcd14e10: 2024-12-08T01:18:45,441 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T01:18:45,442 DEBUG [Time-limited test {}] regionserver.HStore(1541): f837f401ea75f84ac401a4defcd14e10/cf1 is initiating major compaction (all files) 2024-12-08T01:18:45,442 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:18:45,442 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:45,443 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f837f401ea75f84ac401a4defcd14e10/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,443 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/baf541d2fd86407db163d57684d812a1, hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/d3f409a825864a5f96325b4bcd5f9528] into tmpdir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp, totalSize=9.5 K 2024-12-08T01:18:45,444 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting baf541d2fd86407db163d57684d812a1, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733620724592 2024-12-08T01:18:45,445 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d3f409a825864a5f96325b4bcd5f9528, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-08T01:18:45,455 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): f837f401ea75f84ac401a4defcd14e10#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:18:45,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741890_1068 (size=4626) 2024-12-08T01:18:45,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741890_1068 (size=4626) 2024-12-08T01:18:45,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741890_1068 (size=4626) 2024-12-08T01:18:45,470 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf1/c46491759fde4bce86fc3a95515c05e1 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/c46491759fde4bce86fc3a95515c05e1 2024-12-08T01:18:45,483 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in f837f401ea75f84ac401a4defcd14e10/cf1 of f837f401ea75f84ac401a4defcd14e10 into c46491759fde4bce86fc3a95515c05e1(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:18:45,484 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f837f401ea75f84ac401a4defcd14e10: 2024-12-08T01:18:45,484 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-08T01:18:45,484 DEBUG [Time-limited test {}] regionserver.HStore(1541): f837f401ea75f84ac401a4defcd14e10/cf2 is initiating major compaction (all files) 2024-12-08T01:18:45,484 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:18:45,484 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:18:45,484 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f837f401ea75f84ac401a4defcd14e10/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,484 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/e0cd84d480c040428edb8802702b231e] into tmpdir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp, totalSize=4.8 K 2024-12-08T01:18:45,485 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e0cd84d480c040428edb8802702b231e, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-08T01:18:45,490 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): f837f401ea75f84ac401a4defcd14e10#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:18:45,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741891_1069 (size=4592) 2024-12-08T01:18:45,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741891_1069 (size=4592) 2024-12-08T01:18:45,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741891_1069 (size=4592) 2024-12-08T01:18:45,504 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/.tmp/cf2/c9c90d0d3c4147c5927f5a5a95a7a079 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/c9c90d0d3c4147c5927f5a5a95a7a079 2024-12-08T01:18:45,511 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in f837f401ea75f84ac401a4defcd14e10/cf2 of f837f401ea75f84ac401a4defcd14e10 into c9c90d0d3c4147c5927f5a5a95a7a079(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:18:45,511 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f837f401ea75f84ac401a4defcd14e10: 2024-12-08T01:18:45,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f837f401ea75f84ac401a4defcd14e10, source=96982154c400,41905,1733620710058, destination=96982154c400,35607,1733620710216, warming up region on 96982154c400,35607,1733620710216 2024-12-08T01:18:45,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f837f401ea75f84ac401a4defcd14e10, source=96982154c400,41905,1733620710058, destination=96982154c400,35607,1733620710216, running balancer 2024-12-08T01:18:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE 2024-12-08T01:18:45,518 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE 2024-12-08T01:18:45,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(7855): Warmup {ENCODED => f837f401ea75f84ac401a4defcd14e10, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:45,519 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=CLOSING, regionLocation=96982154c400,41905,1733620710058 2024-12-08T01:18:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:45,520 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,521 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf1 2024-12-08T01:18:45,521 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:45,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE because future has completed 2024-12-08T01:18:45,523 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:18:45,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058}] 2024-12-08T01:18:45,530 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/baf541d2fd86407db163d57684d812a1 2024-12-08T01:18:45,535 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/c46491759fde4bce86fc3a95515c05e1 2024-12-08T01:18:45,541 INFO [StoreFileOpener-f837f401ea75f84ac401a4defcd14e10-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d3f409a825864a5f96325b4bcd5f9528 2024-12-08T01:18:45,541 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/d3f409a825864a5f96325b4bcd5f9528 2024-12-08T01:18:45,541 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:45,542 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,543 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf2 2024-12-08T01:18:45,543 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:45,549 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/c9c90d0d3c4147c5927f5a5a95a7a079 2024-12-08T01:18:45,555 INFO [StoreFileOpener-f837f401ea75f84ac401a4defcd14e10-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0cd84d480c040428edb8802702b231e 2024-12-08T01:18:45,555 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/e0cd84d480c040428edb8802702b231e 2024-12-08T01:18:45,556 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(1722): Closing f837f401ea75f84ac401a4defcd14e10, disabling compactions & flushes 2024-12-08T01:18:45,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. after waiting 0 ms 2024-12-08T01:18:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35607 {}] regionserver.HRegion(1676): Region close journal for f837f401ea75f84ac401a4defcd14e10: Waiting for close lock at 1733620725556Disabling compacts and flushes for region at 1733620725556Disabling writes for close at 1733620725556Writing region close event to WAL at 1733620725557 (+1 ms)Closed at 1733620725557 2024-12-08T01:18:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-08T01:18:45,678 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,678 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:18:45,678 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing f837f401ea75f84ac401a4defcd14e10, disabling compactions & flushes 2024-12-08T01:18:45,678 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,678 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,678 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. after waiting 0 ms 2024-12-08T01:18:45,678 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,678 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/baf541d2fd86407db163d57684d812a1, hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/d3f409a825864a5f96325b4bcd5f9528] to archive 2024-12-08T01:18:45,682 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:18:45,686 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/baf541d2fd86407db163d57684d812a1 to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/baf541d2fd86407db163d57684d812a1 2024-12-08T01:18:45,689 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/d3f409a825864a5f96325b4bcd5f9528 to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/d3f409a825864a5f96325b4bcd5f9528 2024-12-08T01:18:45,704 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/e0cd84d480c040428edb8802702b231e] to archive 2024-12-08T01:18:45,705 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:18:45,707 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/e0cd84d480c040428edb8802702b231e to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/e0cd84d480c040428edb8802702b231e 2024-12-08T01:18:45,713 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-08T01:18:45,714 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:45,714 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for f837f401ea75f84ac401a4defcd14e10: Waiting for close lock at 1733620725678Running coprocessor pre-close hooks at 1733620725678Disabling compacts and flushes for region at 1733620725678Disabling writes for close at 1733620725678Writing region close event to WAL at 1733620725709 (+31 ms)Running coprocessor post-close hooks at 1733620725713 (+4 ms)Closed at 1733620725714 (+1 ms) 2024-12-08T01:18:45,714 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding f837f401ea75f84ac401a4defcd14e10 move to 96982154c400,35607,1733620710216 record at close sequenceid=12 2024-12-08T01:18:45,717 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:45,717 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=CLOSED 2024-12-08T01:18:45,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058 because future has completed 2024-12-08T01:18:45,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-08T01:18:45,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058 in 198 msec 2024-12-08T01:18:45,725 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE; state=CLOSED, location=96982154c400,35607,1733620710216; forceNewPlan=false, retain=false 2024-12-08T01:18:45,875 INFO [96982154c400:33573 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T01:18:45,876 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=OPENING, regionLocation=96982154c400,35607,1733620710216 2024-12-08T01:18:45,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE because future has completed 2024-12-08T01:18:45,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216}] 2024-12-08T01:18:46,039 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:46,039 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f837f401ea75f84ac401a4defcd14e10, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:46,039 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,039 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:46,040 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,040 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,041 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,043 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf1 2024-12-08T01:18:46,043 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:46,051 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/c46491759fde4bce86fc3a95515c05e1 2024-12-08T01:18:46,051 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:46,052 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,053 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf2 2024-12-08T01:18:46,053 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:46,059 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/c9c90d0d3c4147c5927f5a5a95a7a079 2024-12-08T01:18:46,059 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:46,060 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,061 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,062 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,063 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,063 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,064 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-08T01:18:46,065 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,067 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened f837f401ea75f84ac401a4defcd14e10; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69613883, jitterRate=0.03732769191265106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-08T01:18:46,067 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,068 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f837f401ea75f84ac401a4defcd14e10: Running coprocessor pre-open hook at 1733620726040Writing region info on filesystem at 1733620726040Initializing all the Stores at 1733620726041 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620726041Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620726041Cleaning up temporary data from old regions at 1733620726063 (+22 ms)Running coprocessor post-open hooks at 1733620726067 (+4 ms)Region opened successfully at 1733620726068 (+1 ms) 2024-12-08T01:18:46,069 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., pid=12, masterSystemTime=1733620726034 2024-12-08T01:18:46,071 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:46,071 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:46,072 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=OPEN, openSeqNum=18, regionLocation=96982154c400,35607,1733620710216 2024-12-08T01:18:46,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216 because future has completed 2024-12-08T01:18:46,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-08T01:18:46,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,35607,1733620710216 in 195 msec 2024-12-08T01:18:46,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, REOPEN/MOVE in 563 msec 2024-12-08T01:18:46,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:18:46,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:18:46,123 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 96982154c400,35607,1733620710216: testing ***** 2024-12-08T01:18:46,123 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-08T01:18:46,126 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-08T01:18:46,128 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-08T01:18:46,131 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-08T01:18:46,133 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-08T01:18:46,141 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 412584024 }, "NonHeapMemoryUsage": { "committed": 170131456, "init": 7667712, "max": -1, "used": 167507224 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "96982154c400", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 11, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 8, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 10, "ProcessCallTime_98th_percentile": 10, "ProcessCallTime_99th_percentile": 10, "ProcessCallTime_99.9th_percentile": 10, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 11, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 8, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 10, "TotalCallTime_98th_percentile": 10, "TotalCallTime_99th_percentile": 10, "TotalCallTime_99.9th_percentile": 10, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "96982154c400", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52656", "tag.serverName": "96982154c400,36455,1733620710167", "tag.clusterId": "ed8c269a-5f9d-4013-89a3-831bf8295dbd", "tag.Context": "regionserver", "tag.Hostname": "96982154c400", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733620710167, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-08T01:18:46,144 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] master.MasterRpcServices(700): 96982154c400,35607,1733620710216 reported a fatal error: ***** ABORTING region server 96982154c400,35607,1733620710216: testing ***** 2024-12-08T01:18:46,147 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '96982154c400,35607,1733620710216' ***** 2024-12-08T01:18:46,147 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-08T01:18:46,147 INFO [RS:2;96982154c400:35607 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T01:18:46,147 INFO [RS:2;96982154c400:35607 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-08T01:18:46,147 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T01:18:46,147 INFO [RS:2;96982154c400:35607 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-08T01:18:46,147 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(3091): Received CLOSE for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,148 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(956): aborting server 96982154c400,35607,1733620710216 2024-12-08T01:18:46,148 INFO [RS:2;96982154c400:35607 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:18:46,148 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f837f401ea75f84ac401a4defcd14e10, disabling compactions & flushes 2024-12-08T01:18:46,148 INFO [RS:2;96982154c400:35607 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;96982154c400:35607. 2024-12-08T01:18:46,148 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:59784 deadline: 1733620786148, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=35607 startCode=1733620710216. As of locationSeqNum=12. 2024-12-08T01:18:46,148 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:46,148 DEBUG [RS:2;96982154c400:35607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:18:46,148 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. after waiting 0 ms 2024-12-08T01:18:46,148 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:46,148 DEBUG [RS:2;96982154c400:35607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:18:46,149 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,41905,1733620710058, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,41905,1733620710058, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=35607 startCode=1733620710216. As of locationSeqNum=12. 2024-12-08T01:18:46,149 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,41905,1733620710058, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=35607 startCode=1733620710216. As of locationSeqNum=12. 2024-12-08T01:18:46,149 INFO [RS:2;96982154c400:35607 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T01:18:46,149 INFO [RS:2;96982154c400:35607 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T01:18:46,149 INFO [RS:2;96982154c400:35607 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T01:18:46,149 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,41905,1733620710058, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=35607 startCode=1733620710216. As of locationSeqNum=12. 2024-12-08T01:18:46,149 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T01:18:46,149 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-08T01:18:46,149 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T01:18:46,149 DEBUG [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1325): Online Regions={f837f401ea75f84ac401a4defcd14e10=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., 1588230740=hbase:meta,,1.1588230740} 2024-12-08T01:18:46,149 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T01:18:46,149 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T01:18:46,150 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T01:18:46,150 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T01:18:46,150 DEBUG [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:46,150 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:46,150 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f837f401ea75f84ac401a4defcd14e10: Waiting for close lock at 1733620726148Running coprocessor pre-close hooks at 1733620726148Disabling compacts and flushes for region at 1733620726148Disabling writes for close at 1733620726148Writing region close event to WAL at 1733620726149 (+1 ms)Running coprocessor post-close hooks at 1733620726150 (+1 ms)Closed at 1733620726150 2024-12-08T01:18:46,150 ERROR [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-08T01:18:46,150 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:46,150 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T01:18:46,150 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T01:18:46,150 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733620726149Running coprocessor pre-close hooks at 1733620726149Disabling compacts and flushes for region at 1733620726149Disabling writes for close at 1733620726150 (+1 ms)Writing region close event to WAL at 1733620726150Running coprocessor post-close hooks at 1733620726150Closed at 1733620726150 2024-12-08T01:18:46,151 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T01:18:46,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35607,1733620710216 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:46,152 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35607,1733620710216 aborting 2024-12-08T01:18:46,152 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35607,1733620710216 aborting 2024-12-08T01:18:46,152 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 from cache 2024-12-08T01:18:46,176 INFO [regionserver/96982154c400:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:18:46,254 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:18:46,256 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1] 2024-12-08T01:18:46,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35607,1733620710216 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:46,258 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35607,1733620710216 aborting 2024-12-08T01:18:46,259 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35607,1733620710216 aborting 2024-12-08T01:18:46,259 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 from cache 2024-12-08T01:18:46,350 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(976): stopping server 96982154c400,35607,1733620710216; all regions closed. 2024-12-08T01:18:46,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741836_1012 (size=6845) 2024-12-08T01:18:46,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741836_1012 (size=6845) 2024-12-08T01:18:46,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741836_1012 (size=6845) 2024-12-08T01:18:46,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741835_1011 (size=2142) 2024-12-08T01:18:46,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741835_1011 (size=2142) 2024-12-08T01:18:46,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741835_1011 (size=2142) 2024-12-08T01:18:46,364 DEBUG [RS:2;96982154c400:35607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:18:46,364 INFO [RS:2;96982154c400:35607 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:18:46,364 INFO [RS:2;96982154c400:35607 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:18:46,364 INFO [RS:2;96982154c400:35607 {}] hbase.ChoreService(370): Chore service for: regionserver/96982154c400:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T01:18:46,365 INFO [RS:2;96982154c400:35607 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:18:46,365 INFO [regionserver/96982154c400:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:18:46,365 INFO [RS:2;96982154c400:35607 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35607 2024-12-08T01:18:46,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:18:46,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/96982154c400,35607,1733620710216 2024-12-08T01:18:46,434 INFO [RS:2;96982154c400:35607 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:18:46,445 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [96982154c400,35607,1733620710216] 2024-12-08T01:18:46,454 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/96982154c400,35607,1733620710216 already deleted, retry=false 2024-12-08T01:18:46,455 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 96982154c400,35607,1733620710216 on 96982154c400,33573,1733620709230 2024-12-08T01:18:46,460 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 96982154c400,35607,1733620710216, splitWal=true, meta=true 2024-12-08T01:18:46,463 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 96982154c400,35607,1733620710216, splitWal=true, meta=true 2024-12-08T01:18:46,463 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 96982154c400,35607,1733620710216 (carryingMeta=true) 96982154c400,35607,1733620710216/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1bcb8c38[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-08T01:18:46,463 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:18:46,465 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1] 2024-12-08T01:18:46,467 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 96982154c400,35607,1733620710216, splitWal=true, meta=true, isMeta: true 2024-12-08T01:18:46,467 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 96982154c400:35607 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 96982154c400/172.17.0.2:35607 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:46,468 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1, error=java.net.ConnectException: Call to address=96982154c400:35607 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 96982154c400/172.17.0.2:35607 2024-12-08T01:18:46,468 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-08T01:18:46,469 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 from cache 2024-12-08T01:18:46,469 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting 2024-12-08T01:18:46,469 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 96982154c400:35607 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 96982154c400/172.17.0.2:35607 2024-12-08T01:18:46,471 INFO [PEWorker-1 {}] master.SplitWALManager(105): 96982154c400,35607,1733620710216 WAL count=1, meta=true 2024-12-08T01:18:46,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta}] 2024-12-08T01:18:46,479 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=96982154c400,41905,1733620710058 2024-12-08T01:18:46,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, worker=96982154c400,41905,1733620710058}] 2024-12-08T01:18:46,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:18:46,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35607-0x1000300bb2c0003, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:18:46,545 INFO [RS:2;96982154c400:35607 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:18:46,545 INFO [RS:2;96982154c400:35607 {}] regionserver.HRegionServer(1031): Exiting; stopping=96982154c400,35607,1733620710216; zookeeper connection closed. 2024-12-08T01:18:46,546 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1a0192b2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1a0192b2 2024-12-08T01:18:46,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-08T01:18:46,663 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, size=6.7 K (6845bytes) 2024-12-08T01:18:46,663 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta 2024-12-08T01:18:46,664 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta after 0ms 2024-12-08T01:18:46,666 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta took 3ms 2024-12-08T01:18:46,671 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-08T01:18:46,672 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta so closing down 2024-12-08T01:18:46,672 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:46,673 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta.temp 2024-12-08T01:18:46,674 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta.temp 2024-12-08T01:18:46,675 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:46,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741892_1070 (size=6604) 2024-12-08T01:18:46,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741892_1070 (size=6604) 2024-12-08T01:18:46,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741892_1070 (size=6604) 2024-12-08T01:18:46,684 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:46,685 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta.temp to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-08T01:18:46,685 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 19 ms; skipped=1; WAL=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-12-08T01:18:46,686 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, journal: Splitting hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, size=6.7 K (6845bytes) at 1733620726663Finishing writing output for hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta so closing down at 1733620726672 (+9 ms)Creating recovered edits writer path=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta.temp at 1733620726674 (+2 ms)3 split writer threads finished at 1733620726675 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733620726684 (+9 ms)Rename recovered edits hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta.temp to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733620726685 (+1 ms)Processed 16 edits across 1 Regions in 19 ms; skipped=1; WAL=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1733620726686 (+1 ms) 2024-12-08T01:18:46,686 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta 2024-12-08T01:18:46,687 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-08T01:18:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-08T01:18:46,692 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs 2024-12-08T01:18:46,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-08T01:18:46,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, worker=96982154c400,41905,1733620710058 in 211 msec 2024-12-08T01:18:46,696 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=96982154c400,41905,1733620710058 2024-12-08T01:18:46,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T01:18:46,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 96982154c400%2C35607%2C1733620710216.meta.1733620712144.meta, worker=96982154c400,41905,1733620710058 in 224 msec 2024-12-08T01:18:46,701 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting dir is empty, no logs to split. 2024-12-08T01:18:46,701 INFO [PEWorker-1 {}] master.SplitWALManager(105): 96982154c400,35607,1733620710216 WAL count=0, meta=true 2024-12-08T01:18:46,701 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 96982154c400,35607,1733620710216 WAL splitting is done? wals=0, meta=true 2024-12-08T01:18:46,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T01:18:46,704 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:18:46,706 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-08T01:18:46,773 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:18:46,775 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1] 2024-12-08T01:18:46,775 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 96982154c400:35607 this server is in the failed servers list 2024-12-08T01:18:46,776 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=96982154c400:35607 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35607 2024-12-08T01:18:46,776 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35607 2024-12-08T01:18:46,776 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 from cache 2024-12-08T01:18:46,856 DEBUG [96982154c400:33573 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-08T01:18:46,856 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(204): Hosts are {96982154c400=0} racks are {/default-rack=0} 2024-12-08T01:18:46,857 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T01:18:46,857 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T01:18:46,857 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T01:18:46,857 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T01:18:46,857 INFO [96982154c400:33573 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T01:18:46,857 INFO [96982154c400:33573 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T01:18:46,857 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T01:18:46,857 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=96982154c400,36455,1733620710167 2024-12-08T01:18:46,859 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,36455,1733620710167, state=OPENING 2024-12-08T01:18:46,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:46,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:46,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:46,897 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:18:46,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:46,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:46,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,36455,1733620710167}] 2024-12-08T01:18:46,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:47,052 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:18:47,053 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42101, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:18:47,058 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T01:18:47,058 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:47,059 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T01:18:47,060 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C36455%2C1733620710167.meta, suffix=.meta, logDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,36455,1733620710167, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs, maxLogs=32 2024-12-08T01:18:47,071 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,36455,1733620710167/96982154c400%2C36455%2C1733620710167.meta.1733620727061.meta, exclude list is [], retry=0 2024-12-08T01:18:47,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:47,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:47,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:47,076 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,36455,1733620710167/96982154c400%2C36455%2C1733620710167.meta.1733620727061.meta 2024-12-08T01:18:47,076 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:47,076 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:47,076 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T01:18:47,077 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T01:18:47,077 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T01:18:47,077 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T01:18:47,077 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:47,077 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T01:18:47,077 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T01:18:47,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T01:18:47,079 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T01:18:47,079 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:47,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:47,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T01:18:47,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T01:18:47,081 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:47,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:47,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T01:18:47,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T01:18:47,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:47,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:47,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T01:18:47,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T01:18:47,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:47,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:18:47,084 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T01:18:47,085 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740 2024-12-08T01:18:47,086 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740 2024-12-08T01:18:47,086 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-08T01:18:47,090 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-08T01:18:47,090 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-08T01:18:47,108 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/info/8c5212888e2d4f74be506672792ec04e is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10./info:regioninfo/1733620726072/Put/seqid=0 2024-12-08T01:18:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741894_1072 (size=11177) 2024-12-08T01:18:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741894_1072 (size=11177) 2024-12-08T01:18:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741894_1072 (size=11177) 2024-12-08T01:18:47,115 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/info/8c5212888e2d4f74be506672792ec04e 2024-12-08T01:18:47,133 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/ns/f72d2c5598e443f7b18374d1e59eea95 is 43, key is default/ns:d/1733620712338/Put/seqid=0 2024-12-08T01:18:47,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741895_1073 (size=5153) 2024-12-08T01:18:47,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741895_1073 (size=5153) 2024-12-08T01:18:47,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741895_1073 (size=5153) 2024-12-08T01:18:47,140 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/ns/f72d2c5598e443f7b18374d1e59eea95 2024-12-08T01:18:47,158 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/table/6972092d606f4e568ee97111f591f176 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733620724353/Put/seqid=0 2024-12-08T01:18:47,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741896_1074 (size=5431) 2024-12-08T01:18:47,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741896_1074 (size=5431) 2024-12-08T01:18:47,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741896_1074 (size=5431) 2024-12-08T01:18:47,165 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/table/6972092d606f4e568ee97111f591f176 2024-12-08T01:18:47,171 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/info/8c5212888e2d4f74be506672792ec04e as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/info/8c5212888e2d4f74be506672792ec04e 2024-12-08T01:18:47,177 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/info/8c5212888e2d4f74be506672792ec04e, entries=36, sequenceid=18, filesize=10.9 K 2024-12-08T01:18:47,178 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/ns/f72d2c5598e443f7b18374d1e59eea95 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/ns/f72d2c5598e443f7b18374d1e59eea95 2024-12-08T01:18:47,184 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/ns/f72d2c5598e443f7b18374d1e59eea95, entries=2, sequenceid=18, filesize=5.0 K 2024-12-08T01:18:47,185 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/table/6972092d606f4e568ee97111f591f176 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/table/6972092d606f4e568ee97111f591f176 2024-12-08T01:18:47,191 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/table/6972092d606f4e568ee97111f591f176, entries=2, sequenceid=18, filesize=5.3 K 2024-12-08T01:18:47,191 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=18, compaction requested=false; wal=null 2024-12-08T01:18:47,192 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-08T01:18:47,193 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T01:18:47,193 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T01:18:47,194 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:18:47,195 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T01:18:47,198 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-08T01:18:47,199 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65486749, jitterRate=-0.02417139708995819}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:18:47,199 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T01:18:47,199 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733620727077Writing region info on filesystem at 1733620727077Initializing all the Stores at 1733620727078 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620727078Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620727078Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620727078Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620727078Obtaining lock to block concurrent updates at 1733620727090 (+12 ms)Preparing flush snapshotting stores in 1588230740 at 1733620727090Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733620727090Flushing stores of hbase:meta,,1.1588230740 at 1733620727090Flushing 1588230740/info: creating writer at 1733620727091 (+1 ms)Flushing 1588230740/info: appending metadata at 1733620727108 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733620727108Flushing 1588230740/ns: creating writer at 1733620727120 (+12 ms)Flushing 1588230740/ns: appending metadata at 1733620727133 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733620727133Flushing 1588230740/table: creating writer at 1733620727146 (+13 ms)Flushing 1588230740/table: appending metadata at 1733620727157 (+11 ms)Flushing 1588230740/table: closing flushed file at 1733620727158 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@576c690f: reopening flushed file at 1733620727170 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@305c6131: reopening flushed file at 1733620727178 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d99af06: reopening flushed file at 1733620727184 (+6 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=18, compaction requested=false; wal=null at 1733620727191 (+7 ms)Cleaning up temporary data from old regions at 1733620727193 (+2 ms)Running coprocessor post-open hooks at 1733620727199 (+6 ms)Region opened successfully at 1733620727199 2024-12-08T01:18:47,200 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733620727052 2024-12-08T01:18:47,203 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T01:18:47,203 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T01:18:47,204 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=96982154c400,36455,1733620710167 2024-12-08T01:18:47,204 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,36455,1733620710167, state=OPEN 2024-12-08T01:18:47,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:47,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:47,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:18:47,223 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=96982154c400,36455,1733620710167 2024-12-08T01:18:47,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:47,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:47,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:18:47,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-08T01:18:47,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,36455,1733620710167 in 326 msec 2024-12-08T01:18:47,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-08T01:18:47,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 524 msec 2024-12-08T01:18:47,229 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 96982154c400,35607,1733620710216 had 2 regions 2024-12-08T01:18:47,230 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 96982154c400,35607,1733620710216, splitWal=true, meta=true, isMeta: false 2024-12-08T01:18:47,232 INFO [PEWorker-2 {}] master.SplitWALManager(105): 96982154c400,35607,1733620710216 WAL count=1, meta=false 2024-12-08T01:18:47,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 96982154c400%2C35607%2C1733620710216.1733620711713}] 2024-12-08T01:18:47,234 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=96982154c400,36455,1733620710167 2024-12-08T01:18:47,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 96982154c400%2C35607%2C1733620710216.1733620711713, worker=96982154c400,36455,1733620710167}] 2024-12-08T01:18:47,283 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:18:47,285 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,36455,1733620710167, seqNum=-1] 2024-12-08T01:18:47,285 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:18:47,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:18:47,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-08T01:18:47,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 96982154c400:35607 this server is in the failed servers list 2024-12-08T01:18:47,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=96982154c400:35607 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35607 2024-12-08T01:18:47,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35607 2024-12-08T01:18:47,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=12 from cache 2024-12-08T01:18:47,411 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713, size=2.1 K (2142bytes) 2024-12-08T01:18:47,411 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713 2024-12-08T01:18:47,411 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713 after 0ms 2024-12-08T01:18:47,413 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713 took 3ms 2024-12-08T01:18:47,416 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for f837f401ea75f84ac401a4defcd14e10: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-08T01:18:47,416 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713 so closing down 2024-12-08T01:18:47,416 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:47,416 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:47,416 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-12-08T01:18:47,416 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713, journal: Splitting hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713, size=2.1 K (2142bytes) at 1733620727411Finishing writing output for hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713 so closing down at 1733620727416 (+5 ms)3 split writer threads finished at 1733620727416Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1733620727416 2024-12-08T01:18:47,416 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713 2024-12-08T01:18:47,416 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-08T01:18:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-08T01:18:47,420 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting/96982154c400%2C35607%2C1733620710216.1733620711713 to hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs 2024-12-08T01:18:47,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-08T01:18:47,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 96982154c400%2C35607%2C1733620710216.1733620711713, worker=96982154c400,36455,1733620710167 in 185 msec 2024-12-08T01:18:47,424 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=96982154c400,36455,1733620710167 2024-12-08T01:18:47,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-08T01:18:47,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 96982154c400%2C35607%2C1733620710216.1733620711713, worker=96982154c400,36455,1733620710167 in 192 msec 2024-12-08T01:18:47,427 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/WALs/96982154c400,35607,1733620710216-splitting dir is empty, no logs to split. 2024-12-08T01:18:47,427 INFO [PEWorker-2 {}] master.SplitWALManager(105): 96982154c400,35607,1733620710216 WAL count=0, meta=false 2024-12-08T01:18:47,427 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 96982154c400,35607,1733620710216 WAL splitting is done? wals=0, meta=false 2024-12-08T01:18:47,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN}] 2024-12-08T01:18:47,431 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN 2024-12-08T01:18:47,432 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-08T01:18:47,583 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(204): Hosts are {96982154c400=0} racks are {/default-rack=0} 2024-12-08T01:18:47,583 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T01:18:47,584 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T01:18:47,584 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T01:18:47,584 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T01:18:47,584 INFO [96982154c400:33573 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T01:18:47,584 INFO [96982154c400:33573 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T01:18:47,584 DEBUG [96982154c400:33573 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T01:18:47,585 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=OPENING, regionLocation=96982154c400,41905,1733620710058 2024-12-08T01:18:47,587 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 96982154c400:35607 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 96982154c400/172.17.0.2:35607 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:47,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1, error=java.net.ConnectException: Call to address=96982154c400:35607 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 96982154c400/172.17.0.2:35607 2024-12-08T01:18:47,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-08T01:18:47,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35607,1733620710216, seqNum=-1 from cache 2024-12-08T01:18:47,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 96982154c400:35607 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 96982154c400/172.17.0.2:35607 2024-12-08T01:18:47,619 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=18] 2024-12-08T01:18:47,619 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 96982154c400:35607 this server is in the failed servers list 2024-12-08T01:18:47,620 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=96982154c400:35607 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35607 2024-12-08T01:18:47,620 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35607 2024-12-08T01:18:47,620 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,35607,1733620710216, seqNum=18 from cache 2024-12-08T01:18:47,694 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:18:47,694 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,36455,1733620710167, seqNum=-1] 2024-12-08T01:18:47,695 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:18:47,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36799, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:18:47,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN because future has completed 2024-12-08T01:18:47,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058}] 2024-12-08T01:18:47,865 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:47,865 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => f837f401ea75f84ac401a4defcd14e10, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:47,866 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,867 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:47,867 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,867 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,869 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,870 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf1 2024-12-08T01:18:47,870 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:47,876 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf1/c46491759fde4bce86fc3a95515c05e1 2024-12-08T01:18:47,876 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:47,876 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,877 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f837f401ea75f84ac401a4defcd14e10 columnFamilyName cf2 2024-12-08T01:18:47,877 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:47,884 DEBUG [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/cf2/c9c90d0d3c4147c5927f5a5a95a7a079 2024-12-08T01:18:47,884 INFO [StoreOpener-f837f401ea75f84ac401a4defcd14e10-1 {}] regionserver.HStore(327): Store=f837f401ea75f84ac401a4defcd14e10/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:47,884 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,885 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,886 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,887 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,887 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,888 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-08T01:18:47,889 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,890 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened f837f401ea75f84ac401a4defcd14e10; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61179479, jitterRate=-0.08835472166538239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-08T01:18:47,890 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:18:47,891 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for f837f401ea75f84ac401a4defcd14e10: Running coprocessor pre-open hook at 1733620727867Writing region info on filesystem at 1733620727867Initializing all the Stores at 1733620727869 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620727869Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620727869Cleaning up temporary data from old regions at 1733620727887 (+18 ms)Running coprocessor post-open hooks at 1733620727890 (+3 ms)Region opened successfully at 1733620727891 (+1 ms) 2024-12-08T01:18:47,892 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., pid=21, masterSystemTime=1733620727856 2024-12-08T01:18:47,894 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:47,894 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:18:47,895 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f837f401ea75f84ac401a4defcd14e10, regionState=OPEN, openSeqNum=18, regionLocation=96982154c400,41905,1733620710058 2024-12-08T01:18:47,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058 because future has completed 2024-12-08T01:18:47,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-08T01:18:47,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure f837f401ea75f84ac401a4defcd14e10, server=96982154c400,41905,1733620710058 in 195 msec 2024-12-08T01:18:47,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-08T01:18:47,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f837f401ea75f84ac401a4defcd14e10, ASSIGN in 471 msec 2024-12-08T01:18:47,904 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 96982154c400,35607,1733620710216 after splitting done 2024-12-08T01:18:47,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 96982154c400,35607,1733620710216, splitWal=true, meta=true in 1.4480 sec 2024-12-08T01:18:47,927 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10., hostname=96982154c400,41905,1733620710058, seqNum=18] 2024-12-08T01:18:47,942 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412 (was 413), OpenFileDescriptor=1050 (was 1021) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 285) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10202 (was 10255) 2024-12-08T01:18:47,944 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-12-08T01:18:47,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=412, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=10201 2024-12-08T01:18:47,956 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-12-08T01:18:47,977 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:47,979 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:47,980 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:47,983 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62399573, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-62399573, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:48,000 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62399573/hregion-62399573.1733620727983, exclude list is [], retry=0 2024-12-08T01:18:48,003 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:48,003 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:48,004 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:48,005 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62399573/hregion-62399573.1733620727983 2024-12-08T01:18:48,006 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:48,006 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2b507047566fd246578942c6e28000e8, NAME => 'testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:48,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741898_1076 (size=67) 2024-12-08T01:18:48,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741898_1076 (size=67) 2024-12-08T01:18:48,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741898_1076 (size=67) 2024-12-08T01:18:48,016 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:48,017 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,019 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName a 2024-12-08T01:18:48,019 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,020 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,020 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,021 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName b 2024-12-08T01:18:48,021 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,022 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,022 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,023 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName c 2024-12-08T01:18:48,023 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,024 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,024 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,026 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,026 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,026 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:48,027 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,029 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:48,030 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2b507047566fd246578942c6e28000e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65986985, jitterRate=-0.016717299818992615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:48,030 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2b507047566fd246578942c6e28000e8: Writing region info on filesystem at 1733620728016Initializing all the Stores at 1733620728017 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728017Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728017Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728017Cleaning up temporary data from old regions at 1733620728026 (+9 ms)Region opened successfully at 1733620728030 (+4 ms) 2024-12-08T01:18:48,030 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2b507047566fd246578942c6e28000e8, disabling compactions & flushes 2024-12-08T01:18:48,030 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. 2024-12-08T01:18:48,030 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. 2024-12-08T01:18:48,030 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. after waiting 0 ms 2024-12-08T01:18:48,030 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. 2024-12-08T01:18:48,031 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. 2024-12-08T01:18:48,031 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2b507047566fd246578942c6e28000e8: Waiting for close lock at 1733620728030Disabling compacts and flushes for region at 1733620728030Disabling writes for close at 1733620728030Writing region close event to WAL at 1733620728031 (+1 ms)Closed at 1733620728031 2024-12-08T01:18:48,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741897_1075 (size=93) 2024-12-08T01:18:48,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741897_1075 (size=93) 2024-12-08T01:18:48,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741897_1075 (size=93) 2024-12-08T01:18:48,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:48,036 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62399573:(num 1733620727983) 2024-12-08T01:18:48,036 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:48,037 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:48,048 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037, exclude list is [], retry=0 2024-12-08T01:18:48,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:48,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:48,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:48,054 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037 2024-12-08T01:18:48,054 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:48,054 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b507047566fd246578942c6e28000e8, NAME => 'testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:48,054 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:48,054 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,054 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,056 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,057 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName a 2024-12-08T01:18:48,057 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,057 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,057 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,058 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName b 2024-12-08T01:18:48,058 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,059 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,059 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,060 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName c 2024-12-08T01:18:48,060 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,060 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,060 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,061 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,063 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,064 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,064 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,064 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:48,066 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,067 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2b507047566fd246578942c6e28000e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60674026, jitterRate=-0.09588655829429626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:48,067 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2b507047566fd246578942c6e28000e8: Writing region info on filesystem at 1733620728054Initializing all the Stores at 1733620728055 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728055Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728056 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728056Cleaning up temporary data from old regions at 1733620728064 (+8 ms)Region opened successfully at 1733620728067 (+3 ms) 2024-12-08T01:18:48,092 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b507047566fd246578942c6e28000e8 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-08T01:18:48,108 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/a/120861af2a0148b8b379249b21a7e0de is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733620728067/Put/seqid=0 2024-12-08T01:18:48,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741900_1078 (size=5958) 2024-12-08T01:18:48,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741900_1078 (size=5958) 2024-12-08T01:18:48,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741900_1078 (size=5958) 2024-12-08T01:18:48,118 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/a/120861af2a0148b8b379249b21a7e0de 2024-12-08T01:18:48,137 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/b/dffccc07fa0049c0be4e060d2a812f1b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733620728075/Put/seqid=0 2024-12-08T01:18:48,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741901_1079 (size=5958) 2024-12-08T01:18:48,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741901_1079 (size=5958) 2024-12-08T01:18:48,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741901_1079 (size=5958) 2024-12-08T01:18:48,144 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/b/dffccc07fa0049c0be4e060d2a812f1b 2024-12-08T01:18:48,169 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/c/66c5544982da472c8b40d2ec0bc4b407 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733620728083/Put/seqid=0 2024-12-08T01:18:48,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741902_1080 (size=5958) 2024-12-08T01:18:48,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741902_1080 (size=5958) 2024-12-08T01:18:48,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741902_1080 (size=5958) 2024-12-08T01:18:48,177 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/c/66c5544982da472c8b40d2ec0bc4b407 2024-12-08T01:18:48,183 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/a/120861af2a0148b8b379249b21a7e0de as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/a/120861af2a0148b8b379249b21a7e0de 2024-12-08T01:18:48,189 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/a/120861af2a0148b8b379249b21a7e0de, entries=10, sequenceid=33, filesize=5.8 K 2024-12-08T01:18:48,190 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/b/dffccc07fa0049c0be4e060d2a812f1b as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/b/dffccc07fa0049c0be4e060d2a812f1b 2024-12-08T01:18:48,196 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/b/dffccc07fa0049c0be4e060d2a812f1b, entries=10, sequenceid=33, filesize=5.8 K 2024-12-08T01:18:48,198 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/c/66c5544982da472c8b40d2ec0bc4b407 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/c/66c5544982da472c8b40d2ec0bc4b407 2024-12-08T01:18:48,204 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/c/66c5544982da472c8b40d2ec0bc4b407, entries=10, sequenceid=33, filesize=5.8 K 2024-12-08T01:18:48,205 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2b507047566fd246578942c6e28000e8 in 114ms, sequenceid=33, compaction requested=false 2024-12-08T01:18:48,205 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2b507047566fd246578942c6e28000e8: 2024-12-08T01:18:48,205 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2b507047566fd246578942c6e28000e8, disabling compactions & flushes 2024-12-08T01:18:48,206 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. 2024-12-08T01:18:48,206 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. 2024-12-08T01:18:48,206 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. after waiting 0 ms 2024-12-08T01:18:48,206 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. 2024-12-08T01:18:48,209 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. 2024-12-08T01:18:48,209 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2b507047566fd246578942c6e28000e8: Waiting for close lock at 1733620728205Disabling compacts and flushes for region at 1733620728205Disabling writes for close at 1733620728206 (+1 ms)Writing region close event to WAL at 1733620728209 (+3 ms)Closed at 1733620728209 2024-12-08T01:18:48,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741899_1077 (size=5937) 2024-12-08T01:18:48,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741899_1077 (size=5937) 2024-12-08T01:18:48,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741899_1077 (size=5937) 2024-12-08T01:18:48,217 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/b/dffccc07fa0049c0be4e060d2a812f1b to hdfs://localhost:46839/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/b/dffccc07fa0049c0be4e060d2a812f1b 2024-12-08T01:18:48,239 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037, size=5.8 K (5937bytes) 2024-12-08T01:18:48,239 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037 2024-12-08T01:18:48,239 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037 after 0ms 2024-12-08T01:18:48,242 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037 took 4ms 2024-12-08T01:18:48,244 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037 so closing down 2024-12-08T01:18:48,244 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:48,245 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620728037.temp 2024-12-08T01:18:48,246 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000003-wal.1733620728037.temp 2024-12-08T01:18:48,246 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:48,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741903_1081 (size=5223) 2024-12-08T01:18:48,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741903_1081 (size=5223) 2024-12-08T01:18:48,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741903_1081 (size=5223) 2024-12-08T01:18:48,255 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000003-wal.1733620728037.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:48,256 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000003-wal.1733620728037.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000032 2024-12-08T01:18:48,256 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-12-08T01:18:48,256 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037, size=5.8 K (5937bytes) at 1733620728239Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037 so closing down at 1733620728244 (+5 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000003-wal.1733620728037.temp at 1733620728246 (+2 ms)3 split writer threads finished at 1733620728246Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000003-wal.1733620728037.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733620728255 (+9 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000003-wal.1733620728037.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000032 at 1733620728256 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1733620728256 2024-12-08T01:18:48,258 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728037 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620728037 2024-12-08T01:18:48,259 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000032 2024-12-08T01:18:48,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:48,261 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:48,272 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728261, exclude list is [], retry=0 2024-12-08T01:18:48,275 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:48,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:48,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:48,278 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728261 2024-12-08T01:18:48,278 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:48,278 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b507047566fd246578942c6e28000e8, NAME => 'testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:48,279 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:48,279 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,279 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,280 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,281 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName a 2024-12-08T01:18:48,281 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,288 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/a/120861af2a0148b8b379249b21a7e0de 2024-12-08T01:18:48,288 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,289 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,290 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName b 2024-12-08T01:18:48,290 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,291 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,291 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,292 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b507047566fd246578942c6e28000e8 columnFamilyName c 2024-12-08T01:18:48,292 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,298 DEBUG [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/c/66c5544982da472c8b40d2ec0bc4b407 2024-12-08T01:18:48,298 INFO [StoreOpener-2b507047566fd246578942c6e28000e8-1 {}] regionserver.HStore(327): Store=2b507047566fd246578942c6e28000e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,298 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,299 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,300 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,301 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000032 2024-12-08T01:18:48,304 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000032 2024-12-08T01:18:48,304 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b507047566fd246578942c6e28000e8 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-08T01:18:48,318 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/b/60ff345dba97461e8894627df0385178 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733620728075/Put/seqid=0 2024-12-08T01:18:48,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741905_1083 (size=5958) 2024-12-08T01:18:48,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741905_1083 (size=5958) 2024-12-08T01:18:48,326 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/b/60ff345dba97461e8894627df0385178 2024-12-08T01:18:48,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741905_1083 (size=5958) 2024-12-08T01:18:48,332 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/.tmp/b/60ff345dba97461e8894627df0385178 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/b/60ff345dba97461e8894627df0385178 2024-12-08T01:18:48,338 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/b/60ff345dba97461e8894627df0385178, entries=10, sequenceid=32, filesize=5.8 K 2024-12-08T01:18:48,338 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2b507047566fd246578942c6e28000e8 in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-12-08T01:18:48,339 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/0000000000000000032 2024-12-08T01:18:48,340 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,340 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,340 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:48,341 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2b507047566fd246578942c6e28000e8 2024-12-08T01:18:48,343 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/2b507047566fd246578942c6e28000e8/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-08T01:18:48,344 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2b507047566fd246578942c6e28000e8; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62921670, jitterRate=-0.06239405274391174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:48,345 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2b507047566fd246578942c6e28000e8: Writing region info on filesystem at 1733620728279Initializing all the Stores at 1733620728280 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728280Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728280Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728280Obtaining lock to block concurrent updates at 1733620728304 (+24 ms)Preparing flush snapshotting stores in 2b507047566fd246578942c6e28000e8 at 1733620728304Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733620728304Flushing stores of testReplayEditsWrittenViaHRegion,,1733620727978.2b507047566fd246578942c6e28000e8. at 1733620728304Flushing 2b507047566fd246578942c6e28000e8/b: creating writer at 1733620728304Flushing 2b507047566fd246578942c6e28000e8/b: appending metadata at 1733620728317 (+13 ms)Flushing 2b507047566fd246578942c6e28000e8/b: closing flushed file at 1733620728317Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a8952ba: reopening flushed file at 1733620728331 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2b507047566fd246578942c6e28000e8 in 34ms, sequenceid=32, compaction requested=false; wal=null at 1733620728338 (+7 ms)Cleaning up temporary data from old regions at 1733620728340 (+2 ms)Region opened successfully at 1733620728345 (+5 ms) 2024-12-08T01:18:48,365 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=422 (was 412) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:42586 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:38184 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:38032 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:58764 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:42794 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:58636 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1126 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 303), ProcessCount=11 (was 11), AvailableMemoryMB=10192 (was 10201) 2024-12-08T01:18:48,365 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-12-08T01:18:48,376 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=10191 2024-12-08T01:18:48,376 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-08T01:18:48,390 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:48,391 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:48,392 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:48,394 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25251693, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-25251693, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:48,404 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25251693/hregion-25251693.1733620728394, exclude list is [], retry=0 2024-12-08T01:18:48,407 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:48,407 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:48,407 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:48,408 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25251693/hregion-25251693.1733620728394 2024-12-08T01:18:48,409 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:18:48,409 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f68cc78296662d012fb1249a02eb670f, NAME => 'testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:48,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741907_1085 (size=68) 2024-12-08T01:18:48,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741907_1085 (size=68) 2024-12-08T01:18:48,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741907_1085 (size=68) 2024-12-08T01:18:48,417 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:48,418 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,419 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName a 2024-12-08T01:18:48,419 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,420 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,420 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,421 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName b 2024-12-08T01:18:48,421 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,421 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,422 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,423 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName c 2024-12-08T01:18:48,423 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,423 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,423 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,424 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,424 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,425 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,425 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,426 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:48,427 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,430 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:48,430 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f68cc78296662d012fb1249a02eb670f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61458066, jitterRate=-0.08420345187187195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:48,431 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f68cc78296662d012fb1249a02eb670f: Writing region info on filesystem at 1733620728417Initializing all the Stores at 1733620728418 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728418Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728418Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728418Cleaning up temporary data from old regions at 1733620728426 (+8 ms)Region opened successfully at 1733620728431 (+5 ms) 2024-12-08T01:18:48,431 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f68cc78296662d012fb1249a02eb670f, disabling compactions & flushes 2024-12-08T01:18:48,431 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:48,431 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:48,431 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. after waiting 0 ms 2024-12-08T01:18:48,431 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:48,432 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:48,432 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f68cc78296662d012fb1249a02eb670f: Waiting for close lock at 1733620728431Disabling compacts and flushes for region at 1733620728431Disabling writes for close at 1733620728431Writing region close event to WAL at 1733620728432 (+1 ms)Closed at 1733620728432 2024-12-08T01:18:48,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741906_1084 (size=93) 2024-12-08T01:18:48,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741906_1084 (size=93) 2024-12-08T01:18:48,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741906_1084 (size=93) 2024-12-08T01:18:48,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:48,439 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25251693:(num 1733620728394) 2024-12-08T01:18:48,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:48,441 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:48,452 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441, exclude list is [], retry=0 2024-12-08T01:18:48,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:48,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:48,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:48,456 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441 2024-12-08T01:18:48,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:48,509 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f68cc78296662d012fb1249a02eb670f, NAME => 'testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:48,512 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,512 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:48,512 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,512 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,514 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,515 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName a 2024-12-08T01:18:48,515 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,516 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,516 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,517 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName b 2024-12-08T01:18:48,517 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,517 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,518 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,518 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName c 2024-12-08T01:18:48,518 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:48,519 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:48,519 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,521 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,521 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,522 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:48,523 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,524 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f68cc78296662d012fb1249a02eb670f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72796468, jitterRate=0.0847519040107727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:48,524 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:48,524 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f68cc78296662d012fb1249a02eb670f: Running coprocessor pre-open hook at 1733620728512Writing region info on filesystem at 1733620728512Initializing all the Stores at 1733620728513 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728514 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728514Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620728514Cleaning up temporary data from old regions at 1733620728521 (+7 ms)Running coprocessor post-open hooks at 1733620728524 (+3 ms)Region opened successfully at 1733620728524 2024-12-08T01:18:48,535 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f68cc78296662d012fb1249a02eb670f 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-08T01:18:48,536 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:49,537 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:49,852 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:18:49,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-08T01:18:49,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:49,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-08T01:18:49,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:49,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-08T01:18:49,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:49,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-08T01:18:49,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:50,538 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:51,540 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:52,541 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:53,543 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:54,544 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:55,545 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:56,546 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:57,547 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f68cc78296662d012fb1249a02eb670f/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:18:57,549 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f68cc78296662d012fb1249a02eb670f: 2024-12-08T01:18:57,549 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:57,563 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f68cc78296662d012fb1249a02eb670f: 2024-12-08T01:18:57,563 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-08T01:18:57,563 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f68cc78296662d012fb1249a02eb670f, disabling compactions & flushes 2024-12-08T01:18:57,563 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:57,563 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:57,563 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. after waiting 0 ms 2024-12-08T01:18:57,563 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:57,564 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:57,564 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. 2024-12-08T01:18:57,564 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f68cc78296662d012fb1249a02eb670f: Waiting for close lock at 1733620737563Running coprocessor pre-close hooks at 1733620737563Disabling compacts and flushes for region at 1733620737563Disabling writes for close at 1733620737563Writing region close event to WAL at 1733620737564 (+1 ms)Running coprocessor post-close hooks at 1733620737564Closed at 1733620737564 2024-12-08T01:18:57,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741908_1086 (size=3941) 2024-12-08T01:18:57,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741908_1086 (size=3941) 2024-12-08T01:18:57,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741908_1086 (size=3941) 2024-12-08T01:18:57,582 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441, size=3.8 K (3941bytes) 2024-12-08T01:18:57,582 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441 2024-12-08T01:18:57,583 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441 after 1ms 2024-12-08T01:18:57,585 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441 took 3ms 2024-12-08T01:18:57,587 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441 so closing down 2024-12-08T01:18:57,587 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:57,588 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733620728441.temp 2024-12-08T01:18:57,589 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000004-wal.1733620728441.temp 2024-12-08T01:18:57,590 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:57,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741909_1087 (size=2983) 2024-12-08T01:18:57,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741909_1087 (size=2983) 2024-12-08T01:18:57,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741909_1087 (size=2983) 2024-12-08T01:18:57,597 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000004-wal.1733620728441.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:57,598 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000004-wal.1733620728441.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000026 2024-12-08T01:18:57,598 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-12-08T01:18:57,598 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441, size=3.8 K (3941bytes) at 1733620737582Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441 so closing down at 1733620737587 (+5 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000004-wal.1733620728441.temp at 1733620737590 (+3 ms)3 split writer threads finished at 1733620737590Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000004-wal.1733620728441.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733620737597 (+7 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000004-wal.1733620728441.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000026 at 1733620737598 (+1 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1733620737598 2024-12-08T01:18:57,600 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620728441 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620728441 2024-12-08T01:18:57,601 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000026 2024-12-08T01:18:57,601 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:57,603 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:57,615 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620737603, exclude list is [], retry=0 2024-12-08T01:18:57,617 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:57,618 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:57,618 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:57,619 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620737603 2024-12-08T01:18:57,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:57,621 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f68cc78296662d012fb1249a02eb670f, NAME => 'testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:57,621 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,621 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:57,621 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,621 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,623 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,624 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName a 2024-12-08T01:18:57,624 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:57,624 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:57,624 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,625 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName b 2024-12-08T01:18:57,625 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:57,626 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:57,626 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,627 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f68cc78296662d012fb1249a02eb670f columnFamilyName c 2024-12-08T01:18:57,627 DEBUG [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:57,627 INFO [StoreOpener-f68cc78296662d012fb1249a02eb670f-1 {}] regionserver.HStore(327): Store=f68cc78296662d012fb1249a02eb670f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:57,627 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,628 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,629 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,630 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000026 2024-12-08T01:18:57,632 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000026 2024-12-08T01:18:57,633 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f68cc78296662d012fb1249a02eb670f 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-08T01:18:57,646 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/a/9f574f5e050c475ba79e4d0d5d6c5dfc is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733620737556/Put/seqid=0 2024-12-08T01:18:57,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741911_1089 (size=5523) 2024-12-08T01:18:57,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741911_1089 (size=5523) 2024-12-08T01:18:57,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741911_1089 (size=5523) 2024-12-08T01:18:57,653 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/a/9f574f5e050c475ba79e4d0d5d6c5dfc 2024-12-08T01:18:57,670 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/b/52ebc73ab9b540d09d661a246d7bafeb is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733620737550/Put/seqid=0 2024-12-08T01:18:57,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741912_1090 (size=5524) 2024-12-08T01:18:57,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741912_1090 (size=5524) 2024-12-08T01:18:57,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741912_1090 (size=5524) 2024-12-08T01:18:57,678 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/b/52ebc73ab9b540d09d661a246d7bafeb 2024-12-08T01:18:57,695 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/c/418821e19e584bf885a4e6626a61db34 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733620737552/Put/seqid=0 2024-12-08T01:18:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741913_1091 (size=5457) 2024-12-08T01:18:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741913_1091 (size=5457) 2024-12-08T01:18:57,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741913_1091 (size=5457) 2024-12-08T01:18:57,702 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/c/418821e19e584bf885a4e6626a61db34 2024-12-08T01:18:57,708 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/a/9f574f5e050c475ba79e4d0d5d6c5dfc as hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/a/9f574f5e050c475ba79e4d0d5d6c5dfc 2024-12-08T01:18:57,714 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/a/9f574f5e050c475ba79e4d0d5d6c5dfc, entries=7, sequenceid=26, filesize=5.4 K 2024-12-08T01:18:57,715 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/b/52ebc73ab9b540d09d661a246d7bafeb as hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/b/52ebc73ab9b540d09d661a246d7bafeb 2024-12-08T01:18:57,720 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/b/52ebc73ab9b540d09d661a246d7bafeb, entries=7, sequenceid=26, filesize=5.4 K 2024-12-08T01:18:57,721 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/.tmp/c/418821e19e584bf885a4e6626a61db34 as hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/c/418821e19e584bf885a4e6626a61db34 2024-12-08T01:18:57,726 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/c/418821e19e584bf885a4e6626a61db34, entries=6, sequenceid=26, filesize=5.3 K 2024-12-08T01:18:57,726 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for f68cc78296662d012fb1249a02eb670f in 93ms, sequenceid=26, compaction requested=false; wal=null 2024-12-08T01:18:57,727 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/0000000000000000026 2024-12-08T01:18:57,728 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,728 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,729 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:57,731 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,733 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/f68cc78296662d012fb1249a02eb670f/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-08T01:18:57,734 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f68cc78296662d012fb1249a02eb670f; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62364350, jitterRate=-0.07069876790046692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:57,734 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f68cc78296662d012fb1249a02eb670f 2024-12-08T01:18:57,734 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f68cc78296662d012fb1249a02eb670f: Running coprocessor pre-open hook at 1733620737621Writing region info on filesystem at 1733620737621Initializing all the Stores at 1733620737622 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620737622Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620737623 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620737623Obtaining lock to block concurrent updates at 1733620737633 (+10 ms)Preparing flush snapshotting stores in f68cc78296662d012fb1249a02eb670f at 1733620737633Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733620737633Flushing stores of testReplayEditsAfterAbortingFlush,,1733620728390.f68cc78296662d012fb1249a02eb670f. at 1733620737633Flushing f68cc78296662d012fb1249a02eb670f/a: creating writer at 1733620737633Flushing f68cc78296662d012fb1249a02eb670f/a: appending metadata at 1733620737646 (+13 ms)Flushing f68cc78296662d012fb1249a02eb670f/a: closing flushed file at 1733620737646Flushing f68cc78296662d012fb1249a02eb670f/b: creating writer at 1733620737658 (+12 ms)Flushing f68cc78296662d012fb1249a02eb670f/b: appending metadata at 1733620737670 (+12 ms)Flushing f68cc78296662d012fb1249a02eb670f/b: closing flushed file at 1733620737670Flushing f68cc78296662d012fb1249a02eb670f/c: creating writer at 1733620737683 (+13 ms)Flushing f68cc78296662d012fb1249a02eb670f/c: appending metadata at 1733620737695 (+12 ms)Flushing f68cc78296662d012fb1249a02eb670f/c: closing flushed file at 1733620737695Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dbf915e: reopening flushed file at 1733620737708 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e21ed87: reopening flushed file at 1733620737714 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f8aa8cf: reopening flushed file at 1733620737720 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for f68cc78296662d012fb1249a02eb670f in 93ms, sequenceid=26, compaction requested=false; wal=null at 1733620737727 (+7 ms)Cleaning up temporary data from old regions at 1733620737728 (+1 ms)Running coprocessor post-open hooks at 1733620737734 (+6 ms)Region opened successfully at 1733620737734 2024-12-08T01:18:57,754 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419 (was 420), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=271 (was 303), ProcessCount=11 (was 11), AvailableMemoryMB=10154 (was 10191) 2024-12-08T01:18:57,754 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-08T01:18:57,765 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=271, ProcessCount=11, AvailableMemoryMB=10153 2024-12-08T01:18:57,765 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-08T01:18:57,779 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:57,781 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:57,781 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:57,784 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-47885066, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-47885066, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:57,796 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-47885066/hregion-47885066.1733620737785, exclude list is [], retry=0 2024-12-08T01:18:57,799 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:57,799 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:57,799 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:57,801 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-47885066/hregion-47885066.1733620737785 2024-12-08T01:18:57,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:18:57,801 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5e3570153bbdf403d42b4a6906ba635b, NAME => 'testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:57,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741915_1093 (size=61) 2024-12-08T01:18:57,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741915_1093 (size=61) 2024-12-08T01:18:57,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741915_1093 (size=61) 2024-12-08T01:18:57,815 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:57,817 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,819 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e3570153bbdf403d42b4a6906ba635b columnFamilyName a 2024-12-08T01:18:57,819 DEBUG [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:57,819 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.HStore(327): Store=5e3570153bbdf403d42b4a6906ba635b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:57,820 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,820 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,821 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,821 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,821 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,823 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,825 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:57,825 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e3570153bbdf403d42b4a6906ba635b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63683532, jitterRate=-0.05104142427444458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:57,826 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e3570153bbdf403d42b4a6906ba635b: Writing region info on filesystem at 1733620737815Initializing all the Stores at 1733620737815Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620737815Cleaning up temporary data from old regions at 1733620737821 (+6 ms)Region opened successfully at 1733620737826 (+5 ms) 2024-12-08T01:18:57,826 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5e3570153bbdf403d42b4a6906ba635b, disabling compactions & flushes 2024-12-08T01:18:57,826 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,826 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,826 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. after waiting 0 ms 2024-12-08T01:18:57,826 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,827 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,827 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5e3570153bbdf403d42b4a6906ba635b: Waiting for close lock at 1733620737826Disabling compacts and flushes for region at 1733620737826Disabling writes for close at 1733620737826Writing region close event to WAL at 1733620737827 (+1 ms)Closed at 1733620737827 2024-12-08T01:18:57,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741914_1092 (size=93) 2024-12-08T01:18:57,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741914_1092 (size=93) 2024-12-08T01:18:57,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741914_1092 (size=93) 2024-12-08T01:18:57,834 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:57,834 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-47885066:(num 1733620737785) 2024-12-08T01:18:57,834 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:57,836 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:57,847 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836, exclude list is [], retry=0 2024-12-08T01:18:57,850 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:57,850 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:57,851 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:57,852 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836 2024-12-08T01:18:57,852 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:57,853 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e3570153bbdf403d42b4a6906ba635b, NAME => 'testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:57,853 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:57,853 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,853 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,854 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,855 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e3570153bbdf403d42b4a6906ba635b columnFamilyName a 2024-12-08T01:18:57,855 DEBUG [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:57,856 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.HStore(327): Store=5e3570153bbdf403d42b4a6906ba635b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:57,856 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,856 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,858 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,858 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,858 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,860 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:57,861 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e3570153bbdf403d42b4a6906ba635b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58774079, jitterRate=-0.12419797480106354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:57,861 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e3570153bbdf403d42b4a6906ba635b: Writing region info on filesystem at 1733620737853Initializing all the Stores at 1733620737854 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620737854Cleaning up temporary data from old regions at 1733620737858 (+4 ms)Region opened successfully at 1733620737861 (+3 ms) 2024-12-08T01:18:57,869 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5e3570153bbdf403d42b4a6906ba635b, disabling compactions & flushes 2024-12-08T01:18:57,869 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,869 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,869 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. after waiting 0 ms 2024-12-08T01:18:57,869 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,870 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,870 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. 2024-12-08T01:18:57,870 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5e3570153bbdf403d42b4a6906ba635b: Waiting for close lock at 1733620737869Disabling compacts and flushes for region at 1733620737869Disabling writes for close at 1733620737869Writing region close event to WAL at 1733620737870 (+1 ms)Closed at 1733620737870 2024-12-08T01:18:57,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741916_1094 (size=1623) 2024-12-08T01:18:57,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741916_1094 (size=1623) 2024-12-08T01:18:57,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741916_1094 (size=1623) 2024-12-08T01:18:57,887 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836, size=1.6 K (1623bytes) 2024-12-08T01:18:57,887 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836 2024-12-08T01:18:57,888 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836 after 1ms 2024-12-08T01:18:57,890 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836 took 3ms 2024-12-08T01:18:57,892 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836 so closing down 2024-12-08T01:18:57,892 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:18:57,894 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620737836.temp 2024-12-08T01:18:57,895 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000003-wal.1733620737836.temp 2024-12-08T01:18:57,895 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:18:57,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741917_1095 (size=1623) 2024-12-08T01:18:57,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741917_1095 (size=1623) 2024-12-08T01:18:57,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741917_1095 (size=1623) 2024-12-08T01:18:57,903 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000003-wal.1733620737836.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-08T01:18:57,904 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000003-wal.1733620737836.temp to hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000012 2024-12-08T01:18:57,904 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-12-08T01:18:57,904 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836, journal: Splitting hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836, size=1.6 K (1623bytes) at 1733620737887Finishing writing output for hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836 so closing down at 1733620737892 (+5 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000003-wal.1733620737836.temp at 1733620737895 (+3 ms)3 split writer threads finished at 1733620737895Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000003-wal.1733620737836.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733620737903 (+8 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000003-wal.1733620737836.temp to hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000012 at 1733620737904 (+1 ms)Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1733620737904 2024-12-08T01:18:57,906 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620737836 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620737836 2024-12-08T01:18:57,907 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000012 2024-12-08T01:18:58,273 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:58,275 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:58,291 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620738276, exclude list is [], retry=0 2024-12-08T01:18:58,294 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:58,295 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:58,295 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:58,297 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620738276 2024-12-08T01:18:58,298 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:18:58,298 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e3570153bbdf403d42b4a6906ba635b, NAME => 'testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:58,298 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:58,298 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,298 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,302 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,304 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e3570153bbdf403d42b4a6906ba635b columnFamilyName a 2024-12-08T01:18:58,304 DEBUG [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:58,305 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.HStore(327): Store=5e3570153bbdf403d42b4a6906ba635b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:58,305 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,307 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,308 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000012 2024-12-08T01:18:58,311 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000012 2024-12-08T01:18:58,311 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5e3570153bbdf403d42b4a6906ba635b 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-08T01:18:58,331 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/.tmp/a/ac203399805141a1805a80d2ef5699f0 is 79, key is testDatalossWhenInputError/a:x0/1733620737861/Put/seqid=0 2024-12-08T01:18:58,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741919_1097 (size=5808) 2024-12-08T01:18:58,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741919_1097 (size=5808) 2024-12-08T01:18:58,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741919_1097 (size=5808) 2024-12-08T01:18:58,340 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/.tmp/a/ac203399805141a1805a80d2ef5699f0 2024-12-08T01:18:58,348 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/.tmp/a/ac203399805141a1805a80d2ef5699f0 as hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/a/ac203399805141a1805a80d2ef5699f0 2024-12-08T01:18:58,357 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/a/ac203399805141a1805a80d2ef5699f0, entries=10, sequenceid=12, filesize=5.7 K 2024-12-08T01:18:58,357 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5e3570153bbdf403d42b4a6906ba635b in 46ms, sequenceid=12, compaction requested=false; wal=null 2024-12-08T01:18:58,358 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/0000000000000000012 2024-12-08T01:18:58,359 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,359 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,362 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,364 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-08T01:18:58,365 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e3570153bbdf403d42b4a6906ba635b; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71534602, jitterRate=0.06594863533973694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:58,365 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e3570153bbdf403d42b4a6906ba635b: Writing region info on filesystem at 1733620738299Initializing all the Stores at 1733620738301 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620738301Obtaining lock to block concurrent updates at 1733620738311 (+10 ms)Preparing flush snapshotting stores in 5e3570153bbdf403d42b4a6906ba635b at 1733620738311Finished memstore snapshotting testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733620738311Flushing stores of testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b. at 1733620738311Flushing 5e3570153bbdf403d42b4a6906ba635b/a: creating writer at 1733620738312 (+1 ms)Flushing 5e3570153bbdf403d42b4a6906ba635b/a: appending metadata at 1733620738331 (+19 ms)Flushing 5e3570153bbdf403d42b4a6906ba635b/a: closing flushed file at 1733620738331Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26ca057c: reopening flushed file at 1733620738347 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5e3570153bbdf403d42b4a6906ba635b in 46ms, sequenceid=12, compaction requested=false; wal=null at 1733620738357 (+10 ms)Cleaning up temporary data from old regions at 1733620738359 (+2 ms)Region opened successfully at 1733620738365 (+6 ms) 2024-12-08T01:18:58,369 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e3570153bbdf403d42b4a6906ba635b, NAME => 'testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:58,369 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733620737779.5e3570153bbdf403d42b4a6906ba635b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:58,369 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,369 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,370 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,371 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e3570153bbdf403d42b4a6906ba635b columnFamilyName a 2024-12-08T01:18:58,371 DEBUG [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:58,377 DEBUG [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/a/ac203399805141a1805a80d2ef5699f0 2024-12-08T01:18:58,377 INFO [StoreOpener-5e3570153bbdf403d42b4a6906ba635b-1 {}] regionserver.HStore(327): Store=5e3570153bbdf403d42b4a6906ba635b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:58,378 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,378 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,380 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,380 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,380 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,382 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e3570153bbdf403d42b4a6906ba635b 2024-12-08T01:18:58,385 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/5e3570153bbdf403d42b4a6906ba635b/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-08T01:18:58,386 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e3570153bbdf403d42b4a6906ba635b; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73278244, jitterRate=0.09193092584609985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:18:58,386 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e3570153bbdf403d42b4a6906ba635b: Writing region info on filesystem at 1733620738369Initializing all the Stores at 1733620738370 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620738370Cleaning up temporary data from old regions at 1733620738380 (+10 ms)Region opened successfully at 1733620738386 (+6 ms) 2024-12-08T01:18:58,397 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T01:18:58,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=431 (was 419) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:43156 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1537938077-172.17.0.2-1733620704924:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:43218 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:33940 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:58808 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:58906 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:33948 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=271 (was 271), ProcessCount=11 (was 11), AvailableMemoryMB=10130 (was 10153) 2024-12-08T01:18:58,409 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-08T01:18:58,421 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=431, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=271, ProcessCount=11, AvailableMemoryMB=10129 2024-12-08T01:18:58,421 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-08T01:18:58,436 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:58,438 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:18:58,438 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:18:58,441 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-04046294, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-04046294, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:58,452 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-04046294/hregion-04046294.1733620738441, exclude list is [], retry=0 2024-12-08T01:18:58,454 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:58,455 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:58,455 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:58,457 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-04046294/hregion-04046294.1733620738441 2024-12-08T01:18:58,458 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:18:58,459 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 617f03c628c939a6ca439a57e59c1e13, NAME => 'testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:18:58,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741921_1099 (size=63) 2024-12-08T01:18:58,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741921_1099 (size=63) 2024-12-08T01:18:58,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741921_1099 (size=63) 2024-12-08T01:18:58,468 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:58,469 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,471 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName a 2024-12-08T01:18:58,471 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:58,471 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:58,472 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,473 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName b 2024-12-08T01:18:58,473 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:58,473 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:58,473 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,474 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName c 2024-12-08T01:18:58,475 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:58,475 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:58,475 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,477 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,477 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,478 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:58,479 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,481 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:18:58,482 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 617f03c628c939a6ca439a57e59c1e13; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62006643, jitterRate=-0.07602901756763458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:58,483 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 617f03c628c939a6ca439a57e59c1e13: Writing region info on filesystem at 1733620738468Initializing all the Stores at 1733620738469 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620738469Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620738469Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620738469Cleaning up temporary data from old regions at 1733620738477 (+8 ms)Region opened successfully at 1733620738483 (+6 ms) 2024-12-08T01:18:58,483 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 617f03c628c939a6ca439a57e59c1e13, disabling compactions & flushes 2024-12-08T01:18:58,483 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:18:58,483 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:18:58,483 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. after waiting 0 ms 2024-12-08T01:18:58,483 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:18:58,484 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:18:58,484 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 617f03c628c939a6ca439a57e59c1e13: Waiting for close lock at 1733620738483Disabling compacts and flushes for region at 1733620738483Disabling writes for close at 1733620738483Writing region close event to WAL at 1733620738483Closed at 1733620738484 (+1 ms) 2024-12-08T01:18:58,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741920_1098 (size=93) 2024-12-08T01:18:58,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741920_1098 (size=93) 2024-12-08T01:18:58,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741920_1098 (size=93) 2024-12-08T01:18:58,490 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:18:58,490 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-04046294:(num 1733620738441) 2024-12-08T01:18:58,490 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:18:58,492 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:18:58,505 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493, exclude list is [], retry=0 2024-12-08T01:18:58,508 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:18:58,508 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:18:58,509 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:18:58,510 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 2024-12-08T01:18:58,510 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:18:58,510 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 617f03c628c939a6ca439a57e59c1e13, NAME => 'testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:18:58,511 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:18:58,511 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,511 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,512 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,513 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName a 2024-12-08T01:18:58,513 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:58,514 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:58,514 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,514 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName b 2024-12-08T01:18:58,514 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:58,515 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:58,515 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,516 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName c 2024-12-08T01:18:58,516 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:18:58,516 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:18:58,516 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,517 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,518 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,520 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,520 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,521 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:18:58,522 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:18:58,523 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 617f03c628c939a6ca439a57e59c1e13; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70100977, jitterRate=0.04458595812320709}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:18:58,524 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 617f03c628c939a6ca439a57e59c1e13: Writing region info on filesystem at 1733620738511Initializing all the Stores at 1733620738512 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620738512Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620738512Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620738512Cleaning up temporary data from old regions at 1733620738520 (+8 ms)Region opened successfully at 1733620738524 (+4 ms) 2024-12-08T01:18:58,528 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733620738528/Put/seqid=0 2024-12-08T01:18:58,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741923_1101 (size=4875) 2024-12-08T01:18:58,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741923_1101 (size=4875) 2024-12-08T01:18:58,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741923_1101 (size=4875) 2024-12-08T01:18:58,538 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733620738538/Put/seqid=0 2024-12-08T01:18:58,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741924_1102 (size=4875) 2024-12-08T01:18:58,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741924_1102 (size=4875) 2024-12-08T01:18:58,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741924_1102 (size=4875) 2024-12-08T01:18:58,551 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733620738551/Put/seqid=0 2024-12-08T01:18:58,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741925_1103 (size=4875) 2024-12-08T01:18:58,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741925_1103 (size=4875) 2024-12-08T01:18:58,557 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 617f03c628c939a6ca439a57e59c1e13/a 2024-12-08T01:18:58,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741925_1103 (size=4875) 2024-12-08T01:18:58,560 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-08T01:18:58,560 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-08T01:18:58,560 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 617f03c628c939a6ca439a57e59c1e13/a 2024-12-08T01:18:58,564 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-08T01:18:58,564 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-08T01:18:58,564 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 617f03c628c939a6ca439a57e59c1e13/a 2024-12-08T01:18:58,567 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-08T01:18:58,567 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-08T01:18:58,568 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 617f03c628c939a6ca439a57e59c1e13 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-08T01:18:58,582 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/.tmp/a/c5b9967c56b047cc92bc1ecb767e95d1 is 55, key is testCompactedBulkLoadedFiles/a:a/1733620738525/Put/seqid=0 2024-12-08T01:18:58,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741926_1104 (size=5107) 2024-12-08T01:18:58,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741926_1104 (size=5107) 2024-12-08T01:18:58,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741926_1104 (size=5107) 2024-12-08T01:18:58,589 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/.tmp/a/c5b9967c56b047cc92bc1ecb767e95d1 2024-12-08T01:18:58,595 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/.tmp/a/c5b9967c56b047cc92bc1ecb767e95d1 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/c5b9967c56b047cc92bc1ecb767e95d1 2024-12-08T01:18:58,599 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/c5b9967c56b047cc92bc1ecb767e95d1, entries=1, sequenceid=4, filesize=5.0 K 2024-12-08T01:18:58,601 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 617f03c628c939a6ca439a57e59c1e13 in 33ms, sequenceid=4, compaction requested=false 2024-12-08T01:18:58,601 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 617f03c628c939a6ca439a57e59c1e13: 2024-12-08T01:18:58,602 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_ 2024-12-08T01:18:58,603 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_ 2024-12-08T01:18:58,604 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_ 2024-12-08T01:18:58,605 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 into 617f03c628c939a6ca439a57e59c1e13/a as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_ - updating store file list. 2024-12-08T01:18:58,610 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:18:58,610 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_ into 617f03c628c939a6ca439a57e59c1e13/a 2024-12-08T01:18:58,610 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 into 617f03c628c939a6ca439a57e59c1e13/a (new location: hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_) 2024-12-08T01:18:58,611 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 into 617f03c628c939a6ca439a57e59c1e13/a as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_ - updating store file list. 2024-12-08T01:18:58,617 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0eda3216c7a5452398e69d6852b81955_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:18:58,617 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_ into 617f03c628c939a6ca439a57e59c1e13/a 2024-12-08T01:18:58,617 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 into 617f03c628c939a6ca439a57e59c1e13/a (new location: hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_) 2024-12-08T01:18:58,618 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 into 617f03c628c939a6ca439a57e59c1e13/a as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_ - updating store file list. 2024-12-08T01:18:58,624 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for fb39de92778546e7973888b6664c7d64_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:18:58,624 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_ into 617f03c628c939a6ca439a57e59c1e13/a 2024-12-08T01:18:58,624 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 into 617f03c628c939a6ca439a57e59c1e13/a (new location: hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_) 2024-12-08T01:18:58,633 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:18:58,633 DEBUG [Time-limited test {}] regionserver.HStore(1541): 617f03c628c939a6ca439a57e59c1e13/a is initiating major compaction (all files) 2024-12-08T01:18:58,633 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 617f03c628c939a6ca439a57e59c1e13/a in testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:18:58,634 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/c5b9967c56b047cc92bc1ecb767e95d1, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_] into tmpdir=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/.tmp, totalSize=19.3 K 2024-12-08T01:18:58,634 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c5b9967c56b047cc92bc1ecb767e95d1, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733620738525 2024-12-08T01:18:58,635 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-08T01:18:58,636 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0eda3216c7a5452398e69d6852b81955_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-08T01:18:58,636 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fb39de92778546e7973888b6664c7d64_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-08T01:18:58,652 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/.tmp/a/61dd82a76eef4730a383640fe31c0870 is 55, key is testCompactedBulkLoadedFiles/a:a/1733620738525/Put/seqid=0 2024-12-08T01:18:58,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741927_1105 (size=6154) 2024-12-08T01:18:58,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741927_1105 (size=6154) 2024-12-08T01:18:58,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741927_1105 (size=6154) 2024-12-08T01:18:58,666 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/.tmp/a/61dd82a76eef4730a383640fe31c0870 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/61dd82a76eef4730a383640fe31c0870 2024-12-08T01:18:58,674 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 617f03c628c939a6ca439a57e59c1e13/a of 617f03c628c939a6ca439a57e59c1e13 into 61dd82a76eef4730a383640fe31c0870(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:18:58,674 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 617f03c628c939a6ca439a57e59c1e13: 2024-12-08T01:18:58,674 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-08T01:18:58,674 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-08T01:18:58,710 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493, size=0 (0bytes) 2024-12-08T01:18:58,710 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 might be still open, length is 0 2024-12-08T01:18:58,710 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 2024-12-08T01:18:58,711 WARN [IPC Server handler 4 on default port 46839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-08T01:18:58,711 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 after 1ms 2024-12-08T01:18:59,552 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-08T01:18:59,552 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-08T01:18:59,553 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-08T01:18:59,553 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:01,330 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:33978 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:45653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33978 dst: /127.0.0.1:45653 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45653 remote=/127.0.0.1:33978]. Total timeout mills is 60000, 57344 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:01,330 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:58948 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39423:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58948 dst: /127.0.0.1:39423 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:01,330 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:43256 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:46125:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43256 dst: /127.0.0.1:46125 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:01,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741922_1106 (size=1659) 2024-12-08T01:19:01,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741922_1106 (size=1659) 2024-12-08T01:19:02,712 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 after 4002ms 2024-12-08T01:19:02,715 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 took 4005ms 2024-12-08T01:19:02,716 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493; continuing. 2024-12-08T01:19:02,716 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 so closing down 2024-12-08T01:19:02,716 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:02,718 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620738493.temp 2024-12-08T01:19:02,719 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000003-wal.1733620738493.temp 2024-12-08T01:19:02,719 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:02,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741928_1107 (size=698) 2024-12-08T01:19:02,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741928_1107 (size=698) 2024-12-08T01:19:02,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741928_1107 (size=698) 2024-12-08T01:19:02,726 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000003-wal.1733620738493.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:02,727 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000003-wal.1733620738493.temp to hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000008 2024-12-08T01:19:02,727 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493, size=0, length=0, corrupted=false, cancelled=false 2024-12-08T01:19:02,727 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493, journal: Splitting hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493, size=0 (0bytes) at 1733620738710Finishing writing output for hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 so closing down at 1733620742716 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000003-wal.1733620738493.temp at 1733620742719 (+3 ms)3 split writer threads finished at 1733620742719Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000003-wal.1733620738493.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733620742726 (+7 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000003-wal.1733620738493.temp to hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000008 at 1733620742727 (+1 ms)Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493, size=0, length=0, corrupted=false, cancelled=false at 1733620742727 2024-12-08T01:19:02,729 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620738493 2024-12-08T01:19:02,730 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000008 2024-12-08T01:19:02,730 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:02,731 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:02,742 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620742732, exclude list is [], retry=0 2024-12-08T01:19:02,745 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:19:02,745 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:19:02,745 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:19:02,747 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620742732 2024-12-08T01:19:02,747 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499)] 2024-12-08T01:19:02,747 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 617f03c628c939a6ca439a57e59c1e13, NAME => 'testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:02,747 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:02,747 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,747 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,749 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,749 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName a 2024-12-08T01:19:02,749 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,755 DEBUG [StoreFileOpener-617f03c628c939a6ca439a57e59c1e13-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0eda3216c7a5452398e69d6852b81955_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:02,756 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_ 2024-12-08T01:19:02,759 DEBUG [StoreFileOpener-617f03c628c939a6ca439a57e59c1e13-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:02,759 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_ 2024-12-08T01:19:02,762 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/61dd82a76eef4730a383640fe31c0870 2024-12-08T01:19:02,765 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/c5b9967c56b047cc92bc1ecb767e95d1 2024-12-08T01:19:02,769 DEBUG [StoreFileOpener-617f03c628c939a6ca439a57e59c1e13-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for fb39de92778546e7973888b6664c7d64_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:02,769 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_ 2024-12-08T01:19:02,769 WARN [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7851c87 2024-12-08T01:19:02,769 WARN [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7851c87 2024-12-08T01:19:02,769 WARN [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/c5b9967c56b047cc92bc1ecb767e95d1 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7851c87 2024-12-08T01:19:02,769 WARN [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7851c87 2024-12-08T01:19:02,769 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/c5b9967c56b047cc92bc1ecb767e95d1, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_] to archive 2024-12-08T01:19:02,770 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:19:02,772 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_ to hdfs://localhost:46839/hbase/archive/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/0eda3216c7a5452398e69d6852b81955_SeqId_4_ 2024-12-08T01:19:02,773 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_ to hdfs://localhost:46839/hbase/archive/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_ 2024-12-08T01:19:02,775 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/c5b9967c56b047cc92bc1ecb767e95d1 to hdfs://localhost:46839/hbase/archive/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/c5b9967c56b047cc92bc1ecb767e95d1 2024-12-08T01:19:02,775 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_ to hdfs://localhost:46839/hbase/archive/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/a/fb39de92778546e7973888b6664c7d64_SeqId_4_ 2024-12-08T01:19:02,776 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,776 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,777 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName b 2024-12-08T01:19:02,777 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,777 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,777 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,778 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 617f03c628c939a6ca439a57e59c1e13 columnFamilyName c 2024-12-08T01:19:02,778 DEBUG [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,778 INFO [StoreOpener-617f03c628c939a6ca439a57e59c1e13-1 {}] regionserver.HStore(327): Store=617f03c628c939a6ca439a57e59c1e13/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,779 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,779 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,780 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,781 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000008 2024-12-08T01:19:02,783 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 617f03c628c939a6ca439a57e59c1e13 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "617f03c628c939a6ca439a57e59c1e13" family_name: "a" compaction_input: "c5b9967c56b047cc92bc1ecb767e95d1" compaction_input: "4dc5f8300b7e47f2b6e11fd0396ea020_SeqId_4_" compaction_input: "0eda3216c7a5452398e69d6852b81955_SeqId_4_" compaction_input: "fb39de92778546e7973888b6664c7d64_SeqId_4_" compaction_output: "61dd82a76eef4730a383640fe31c0870" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-08T01:19:02,783 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-08T01:19:02,783 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000008 2024-12-08T01:19:02,784 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/0000000000000000008 2024-12-08T01:19:02,785 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,785 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,786 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:02,787 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 617f03c628c939a6ca439a57e59c1e13 2024-12-08T01:19:02,789 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/617f03c628c939a6ca439a57e59c1e13/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-08T01:19:02,790 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 617f03c628c939a6ca439a57e59c1e13; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74224944, jitterRate=0.10603785514831543}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:02,790 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 617f03c628c939a6ca439a57e59c1e13: Writing region info on filesystem at 1733620742747Initializing all the Stores at 1733620742748 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742748Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742748Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742748Cleaning up temporary data from old regions at 1733620742785 (+37 ms)Region opened successfully at 1733620742790 (+5 ms) 2024-12-08T01:19:02,792 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 617f03c628c939a6ca439a57e59c1e13, disabling compactions & flushes 2024-12-08T01:19:02,792 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:19:02,792 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:19:02,792 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. after waiting 0 ms 2024-12-08T01:19:02,792 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:19:02,793 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733620738437.617f03c628c939a6ca439a57e59c1e13. 2024-12-08T01:19:02,793 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 617f03c628c939a6ca439a57e59c1e13: Waiting for close lock at 1733620742792Disabling compacts and flushes for region at 1733620742792Disabling writes for close at 1733620742792Writing region close event to WAL at 1733620742793 (+1 ms)Closed at 1733620742793 2024-12-08T01:19:02,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741929_1108 (size=93) 2024-12-08T01:19:02,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741929_1108 (size=93) 2024-12-08T01:19:02,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741929_1108 (size=93) 2024-12-08T01:19:02,797 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:02,797 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620742732) 2024-12-08T01:19:02,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=444 (was 431) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-822811002_22 at /127.0.0.1:36238 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:46839 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-822811002_22 at /127.0.0.1:41170 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-822811002_22 at /127.0.0.1:37750 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:46839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=249 (was 271), ProcessCount=11 (was 11), AvailableMemoryMB=10002 (was 10129) 2024-12-08T01:19:02,809 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-08T01:19:02,820 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=249, ProcessCount=11, AvailableMemoryMB=10001 2024-12-08T01:19:02,820 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-08T01:19:02,833 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:02,835 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:02,836 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:02,838 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-83576942, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-83576942, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:02,849 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-83576942/hregion-83576942.1733620742839, exclude list is [], retry=0 2024-12-08T01:19:02,852 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:19:02,852 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:19:02,852 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:19:02,854 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-83576942/hregion-83576942.1733620742839 2024-12-08T01:19:02,854 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:19:02,854 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 919b9404dbe18a91c95c90b6e7a92be8, NAME => 'testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-12-08T01:19:02,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741931_1110 (size=67) 2024-12-08T01:19:02,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741931_1110 (size=67) 2024-12-08T01:19:02,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741931_1110 (size=67) 2024-12-08T01:19:02,862 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:02,863 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,864 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName a 2024-12-08T01:19:02,864 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,864 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,864 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,865 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName b 2024-12-08T01:19:02,865 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,866 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,866 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,867 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName c 2024-12-08T01:19:02,867 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,867 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,867 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,869 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,869 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,869 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:02,870 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,872 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:02,872 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 919b9404dbe18a91c95c90b6e7a92be8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72129920, jitterRate=0.07481956481933594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:02,872 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 919b9404dbe18a91c95c90b6e7a92be8: Writing region info on filesystem at 1733620742862Initializing all the Stores at 1733620742863 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742863Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742863Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742863Cleaning up temporary data from old regions at 1733620742869 (+6 ms)Region opened successfully at 1733620742872 (+3 ms) 2024-12-08T01:19:02,873 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 919b9404dbe18a91c95c90b6e7a92be8, disabling compactions & flushes 2024-12-08T01:19:02,873 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,873 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,873 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. after waiting 0 ms 2024-12-08T01:19:02,873 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,873 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,873 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 919b9404dbe18a91c95c90b6e7a92be8: Waiting for close lock at 1733620742872Disabling compacts and flushes for region at 1733620742872Disabling writes for close at 1733620742873 (+1 ms)Writing region close event to WAL at 1733620742873Closed at 1733620742873 2024-12-08T01:19:02,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741930_1109 (size=93) 2024-12-08T01:19:02,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741930_1109 (size=93) 2024-12-08T01:19:02,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741930_1109 (size=93) 2024-12-08T01:19:02,877 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:02,877 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-83576942:(num 1733620742839) 2024-12-08T01:19:02,877 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:02,879 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:02,890 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879, exclude list is [], retry=0 2024-12-08T01:19:02,893 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:19:02,893 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:19:02,893 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:19:02,895 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879 2024-12-08T01:19:02,895 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469),(127.0.0.1/127.0.0.1:45421:45421)] 2024-12-08T01:19:02,895 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 919b9404dbe18a91c95c90b6e7a92be8, NAME => 'testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:02,895 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:02,895 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,895 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,896 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,897 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName a 2024-12-08T01:19:02,897 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,898 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,898 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,899 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName b 2024-12-08T01:19:02,899 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,899 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,899 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,900 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName c 2024-12-08T01:19:02,900 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:02,900 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:02,900 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,902 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,903 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:02,904 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:02,905 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 919b9404dbe18a91c95c90b6e7a92be8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74287262, jitterRate=0.10696646571159363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:02,905 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 919b9404dbe18a91c95c90b6e7a92be8: Writing region info on filesystem at 1733620742895Initializing all the Stores at 1733620742896 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742896Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742896Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620742896Cleaning up temporary data from old regions at 1733620742903 (+7 ms)Region opened successfully at 1733620742905 (+2 ms) 2024-12-08T01:19:02,912 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 919b9404dbe18a91c95c90b6e7a92be8 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-08T01:19:02,925 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/a/10d8466cf0ae4fcdac5d64de3b52b0f5 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733620742905/Put/seqid=0 2024-12-08T01:19:02,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741933_1112 (size=5958) 2024-12-08T01:19:02,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741933_1112 (size=5958) 2024-12-08T01:19:02,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741933_1112 (size=5958) 2024-12-08T01:19:02,932 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/a/10d8466cf0ae4fcdac5d64de3b52b0f5 2024-12-08T01:19:02,937 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/a/10d8466cf0ae4fcdac5d64de3b52b0f5 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/a/10d8466cf0ae4fcdac5d64de3b52b0f5 2024-12-08T01:19:02,942 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/a/10d8466cf0ae4fcdac5d64de3b52b0f5, entries=10, sequenceid=13, filesize=5.8 K 2024-12-08T01:19:02,943 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 919b9404dbe18a91c95c90b6e7a92be8 in 31ms, sequenceid=13, compaction requested=false 2024-12-08T01:19:02,943 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 919b9404dbe18a91c95c90b6e7a92be8: 2024-12-08T01:19:02,957 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 919b9404dbe18a91c95c90b6e7a92be8, disabling compactions & flushes 2024-12-08T01:19:02,958 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,958 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,958 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. after waiting 0 ms 2024-12-08T01:19:02,958 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,958 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,958 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:02,958 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 919b9404dbe18a91c95c90b6e7a92be8: Waiting for close lock at 1733620742957Disabling compacts and flushes for region at 1733620742957Disabling writes for close at 1733620742958 (+1 ms)Writing region close event to WAL at 1733620742958Closed at 1733620742958 2024-12-08T01:19:02,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741932_1111 (size=5869) 2024-12-08T01:19:02,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741932_1111 (size=5869) 2024-12-08T01:19:02,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741932_1111 (size=5869) 2024-12-08T01:19:02,975 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879, size=5.7 K (5869bytes) 2024-12-08T01:19:02,975 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879 2024-12-08T01:19:02,975 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879 after 0ms 2024-12-08T01:19:02,977 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879 took 2ms 2024-12-08T01:19:02,979 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879 so closing down 2024-12-08T01:19:02,979 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:02,981 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620742879.temp 2024-12-08T01:19:02,982 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000003-wal.1733620742879.temp 2024-12-08T01:19:02,982 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:02,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741934_1113 (size=5223) 2024-12-08T01:19:02,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741934_1113 (size=5223) 2024-12-08T01:19:02,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741934_1113 (size=5223) 2024-12-08T01:19:02,989 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000003-wal.1733620742879.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:02,990 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000003-wal.1733620742879.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000035 2024-12-08T01:19:02,990 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-12-08T01:19:02,990 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879, size=5.7 K (5869bytes) at 1733620742975Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879 so closing down at 1733620742979 (+4 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000003-wal.1733620742879.temp at 1733620742982 (+3 ms)3 split writer threads finished at 1733620742982Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000003-wal.1733620742879.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733620742989 (+7 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000003-wal.1733620742879.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000035 at 1733620742990 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1733620742990 2024-12-08T01:19:02,992 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742879 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620742879 2024-12-08T01:19:02,992 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000035 2024-12-08T01:19:02,993 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:02,994 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:03,005 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994, exclude list is [], retry=0 2024-12-08T01:19:03,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:19:03,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:19:03,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:19:03,010 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 2024-12-08T01:19:03,010 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:19:03,010 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 919b9404dbe18a91c95c90b6e7a92be8, NAME => 'testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:03,010 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:03,011 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,011 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,012 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,013 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName a 2024-12-08T01:19:03,013 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:03,021 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/a/10d8466cf0ae4fcdac5d64de3b52b0f5 2024-12-08T01:19:03,021 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:03,022 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,022 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName b 2024-12-08T01:19:03,022 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:03,023 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:03,023 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,024 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName c 2024-12-08T01:19:03,024 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:03,024 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:03,024 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,027 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000035 2024-12-08T01:19:03,030 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000035 2024-12-08T01:19:03,030 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 919b9404dbe18a91c95c90b6e7a92be8 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-08T01:19:03,043 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/b/ce2db4aadc8241bcab25af4a3f1f85f9 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733620742943/Put/seqid=0 2024-12-08T01:19:03,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741936_1115 (size=5958) 2024-12-08T01:19:03,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741936_1115 (size=5958) 2024-12-08T01:19:03,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741936_1115 (size=5958) 2024-12-08T01:19:03,050 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/b/ce2db4aadc8241bcab25af4a3f1f85f9 2024-12-08T01:19:03,066 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/c/9f8536e241124f83a72cb9154c2be4be is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733620742949/Put/seqid=0 2024-12-08T01:19:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741937_1116 (size=5958) 2024-12-08T01:19:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741937_1116 (size=5958) 2024-12-08T01:19:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741937_1116 (size=5958) 2024-12-08T01:19:03,072 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/c/9f8536e241124f83a72cb9154c2be4be 2024-12-08T01:19:03,077 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/b/ce2db4aadc8241bcab25af4a3f1f85f9 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/b/ce2db4aadc8241bcab25af4a3f1f85f9 2024-12-08T01:19:03,081 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/b/ce2db4aadc8241bcab25af4a3f1f85f9, entries=10, sequenceid=35, filesize=5.8 K 2024-12-08T01:19:03,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/c/9f8536e241124f83a72cb9154c2be4be as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/c/9f8536e241124f83a72cb9154c2be4be 2024-12-08T01:19:03,087 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/c/9f8536e241124f83a72cb9154c2be4be, entries=10, sequenceid=35, filesize=5.8 K 2024-12-08T01:19:03,087 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 919b9404dbe18a91c95c90b6e7a92be8 in 57ms, sequenceid=35, compaction requested=false; wal=null 2024-12-08T01:19:03,088 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000035 2024-12-08T01:19:03,089 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,089 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,089 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:03,090 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:03,092 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-08T01:19:03,093 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 919b9404dbe18a91c95c90b6e7a92be8; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59503097, jitterRate=-0.1133347600698471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:03,093 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 919b9404dbe18a91c95c90b6e7a92be8: Writing region info on filesystem at 1733620743011Initializing all the Stores at 1733620743012 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620743012Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620743012Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620743012Obtaining lock to block concurrent updates at 1733620743030 (+18 ms)Preparing flush snapshotting stores in 919b9404dbe18a91c95c90b6e7a92be8 at 1733620743030Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733620743030Flushing stores of testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. at 1733620743030Flushing 919b9404dbe18a91c95c90b6e7a92be8/b: creating writer at 1733620743030Flushing 919b9404dbe18a91c95c90b6e7a92be8/b: appending metadata at 1733620743043 (+13 ms)Flushing 919b9404dbe18a91c95c90b6e7a92be8/b: closing flushed file at 1733620743043Flushing 919b9404dbe18a91c95c90b6e7a92be8/c: creating writer at 1733620743054 (+11 ms)Flushing 919b9404dbe18a91c95c90b6e7a92be8/c: appending metadata at 1733620743065 (+11 ms)Flushing 919b9404dbe18a91c95c90b6e7a92be8/c: closing flushed file at 1733620743065Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bbf2341: reopening flushed file at 1733620743076 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5998fde3: reopening flushed file at 1733620743081 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 919b9404dbe18a91c95c90b6e7a92be8 in 57ms, sequenceid=35, compaction requested=false; wal=null at 1733620743087 (+6 ms)Cleaning up temporary data from old regions at 1733620743089 (+2 ms)Region opened successfully at 1733620743093 (+4 ms) 2024-12-08T01:19:03,106 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:19:03,153 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994, size=0 (0bytes) 2024-12-08T01:19:03,153 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 might be still open, length is 0 2024-12-08T01:19:03,153 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 2024-12-08T01:19:03,154 WARN [IPC Server handler 0 on default port 46839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-08T01:19:03,154 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 after 1ms 2024-12-08T01:19:04,333 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:41228 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:45653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41228 dst: /127.0.0.1:45653 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45653 remote=/127.0.0.1:41228]. Total timeout mills is 60000, 58785 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:04,334 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:37812 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39423:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37812 dst: /127.0.0.1:39423 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:04,335 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1774669608_22 at /127.0.0.1:36302 [Receiving block BP-1537938077-172.17.0.2-1733620704924:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:46125:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36302 dst: /127.0.0.1:46125 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:04,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741935_1117 (size=5215) 2024-12-08T01:19:04,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741935_1117 (size=5215) 2024-12-08T01:19:07,156 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 after 4003ms 2024-12-08T01:19:07,162 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 took 4008ms 2024-12-08T01:19:07,165 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994; continuing. 2024-12-08T01:19:07,165 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 so closing down 2024-12-08T01:19:07,165 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:07,167 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733620742994.temp 2024-12-08T01:19:07,169 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000037-wal.1733620742994.temp 2024-12-08T01:19:07,169 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:07,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741938_1118 (size=5223) 2024-12-08T01:19:07,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741938_1118 (size=5223) 2024-12-08T01:19:07,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741938_1118 (size=5223) 2024-12-08T01:19:07,176 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000037-wal.1733620742994.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:07,177 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000037-wal.1733620742994.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000066 2024-12-08T01:19:07,177 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994, size=0, length=0, corrupted=false, cancelled=false 2024-12-08T01:19:07,177 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994, size=0 (0bytes) at 1733620743153Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 so closing down at 1733620747165 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000037-wal.1733620742994.temp at 1733620747169 (+4 ms)3 split writer threads finished at 1733620747169Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000037-wal.1733620742994.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733620747176 (+7 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000037-wal.1733620742994.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000066 at 1733620747177 (+1 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994, size=0, length=0, corrupted=false, cancelled=false at 1733620747177 2024-12-08T01:19:07,179 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 to hdfs://localhost:46839/hbase/oldWALs/wal.1733620742994 2024-12-08T01:19:07,179 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000066 2024-12-08T01:19:07,179 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:07,181 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:07,193 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620747181, exclude list is [], retry=0 2024-12-08T01:19:07,195 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45653,DS-ca67abd9-9abf-444b-b8be-d54e57b5b322,DISK] 2024-12-08T01:19:07,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46125,DS-cbf4ec73-a8e6-4441-aaff-b7054ca52d8b,DISK] 2024-12-08T01:19:07,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39423,DS-c6f66265-baf1-4ce1-88c9-c0680d1d506e,DISK] 2024-12-08T01:19:07,197 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620747181 2024-12-08T01:19:07,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:46499:46499),(127.0.0.1/127.0.0.1:35469:35469)] 2024-12-08T01:19:07,198 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:07,200 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,200 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName a 2024-12-08T01:19:07,201 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:07,206 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/a/10d8466cf0ae4fcdac5d64de3b52b0f5 2024-12-08T01:19:07,206 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:07,206 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,207 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName b 2024-12-08T01:19:07,207 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:07,212 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/b/ce2db4aadc8241bcab25af4a3f1f85f9 2024-12-08T01:19:07,212 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:07,212 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,213 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 919b9404dbe18a91c95c90b6e7a92be8 columnFamilyName c 2024-12-08T01:19:07,213 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:07,218 DEBUG [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/c/9f8536e241124f83a72cb9154c2be4be 2024-12-08T01:19:07,218 INFO [StoreOpener-919b9404dbe18a91c95c90b6e7a92be8-1 {}] regionserver.HStore(327): Store=919b9404dbe18a91c95c90b6e7a92be8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:07,218 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,220 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,220 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000066 2024-12-08T01:19:07,225 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000066 2024-12-08T01:19:07,225 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 919b9404dbe18a91c95c90b6e7a92be8 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-08T01:19:07,238 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/a/4fd154c465794bf8a947e525247854b2 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733620743099/Put/seqid=0 2024-12-08T01:19:07,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741940_1120 (size=5958) 2024-12-08T01:19:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741940_1120 (size=5958) 2024-12-08T01:19:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741940_1120 (size=5958) 2024-12-08T01:19:07,245 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/a/4fd154c465794bf8a947e525247854b2 2024-12-08T01:19:07,263 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/b/972ad694120047708e9af3735ffe586f is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733620743105/Put/seqid=0 2024-12-08T01:19:07,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741941_1121 (size=5958) 2024-12-08T01:19:07,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741941_1121 (size=5958) 2024-12-08T01:19:07,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741941_1121 (size=5958) 2024-12-08T01:19:07,269 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/b/972ad694120047708e9af3735ffe586f 2024-12-08T01:19:07,287 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/c/72bd226b9e7e4b8d9f9400a0f92a043d is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733620743111/Put/seqid=0 2024-12-08T01:19:07,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741942_1122 (size=5958) 2024-12-08T01:19:07,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741942_1122 (size=5958) 2024-12-08T01:19:07,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741942_1122 (size=5958) 2024-12-08T01:19:07,294 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/c/72bd226b9e7e4b8d9f9400a0f92a043d 2024-12-08T01:19:07,299 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/a/4fd154c465794bf8a947e525247854b2 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/a/4fd154c465794bf8a947e525247854b2 2024-12-08T01:19:07,304 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/a/4fd154c465794bf8a947e525247854b2, entries=10, sequenceid=66, filesize=5.8 K 2024-12-08T01:19:07,305 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/b/972ad694120047708e9af3735ffe586f as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/b/972ad694120047708e9af3735ffe586f 2024-12-08T01:19:07,310 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/b/972ad694120047708e9af3735ffe586f, entries=10, sequenceid=66, filesize=5.8 K 2024-12-08T01:19:07,311 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/.tmp/c/72bd226b9e7e4b8d9f9400a0f92a043d as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/c/72bd226b9e7e4b8d9f9400a0f92a043d 2024-12-08T01:19:07,316 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/c/72bd226b9e7e4b8d9f9400a0f92a043d, entries=10, sequenceid=66, filesize=5.8 K 2024-12-08T01:19:07,316 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 919b9404dbe18a91c95c90b6e7a92be8 in 91ms, sequenceid=66, compaction requested=false; wal=null 2024-12-08T01:19:07,317 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/0000000000000000066 2024-12-08T01:19:07,318 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,318 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,318 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:07,320 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 919b9404dbe18a91c95c90b6e7a92be8 2024-12-08T01:19:07,322 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/919b9404dbe18a91c95c90b6e7a92be8/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-08T01:19:07,323 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 919b9404dbe18a91c95c90b6e7a92be8; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59334294, jitterRate=-0.11585012078285217}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:07,323 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 919b9404dbe18a91c95c90b6e7a92be8: Writing region info on filesystem at 1733620747198Initializing all the Stores at 1733620747199 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620747199Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620747199Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620747199Obtaining lock to block concurrent updates at 1733620747225 (+26 ms)Preparing flush snapshotting stores in 919b9404dbe18a91c95c90b6e7a92be8 at 1733620747225Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733620747226 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. at 1733620747226Flushing 919b9404dbe18a91c95c90b6e7a92be8/a: creating writer at 1733620747226Flushing 919b9404dbe18a91c95c90b6e7a92be8/a: appending metadata at 1733620747238 (+12 ms)Flushing 919b9404dbe18a91c95c90b6e7a92be8/a: closing flushed file at 1733620747238Flushing 919b9404dbe18a91c95c90b6e7a92be8/b: creating writer at 1733620747249 (+11 ms)Flushing 919b9404dbe18a91c95c90b6e7a92be8/b: appending metadata at 1733620747262 (+13 ms)Flushing 919b9404dbe18a91c95c90b6e7a92be8/b: closing flushed file at 1733620747262Flushing 919b9404dbe18a91c95c90b6e7a92be8/c: creating writer at 1733620747274 (+12 ms)Flushing 919b9404dbe18a91c95c90b6e7a92be8/c: appending metadata at 1733620747286 (+12 ms)Flushing 919b9404dbe18a91c95c90b6e7a92be8/c: closing flushed file at 1733620747286Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@604b1a7c: reopening flushed file at 1733620747298 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44627350: reopening flushed file at 1733620747304 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33949630: reopening flushed file at 1733620747310 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 919b9404dbe18a91c95c90b6e7a92be8 in 91ms, sequenceid=66, compaction requested=false; wal=null at 1733620747316 (+6 ms)Cleaning up temporary data from old regions at 1733620747318 (+2 ms)Region opened successfully at 1733620747323 (+5 ms) 2024-12-08T01:19:07,335 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 919b9404dbe18a91c95c90b6e7a92be8, disabling compactions & flushes 2024-12-08T01:19:07,335 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:07,335 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:07,335 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. after waiting 0 ms 2024-12-08T01:19:07,335 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:07,336 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620742834.919b9404dbe18a91c95c90b6e7a92be8. 2024-12-08T01:19:07,336 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 919b9404dbe18a91c95c90b6e7a92be8: Waiting for close lock at 1733620747335Disabling compacts and flushes for region at 1733620747335Disabling writes for close at 1733620747335Writing region close event to WAL at 1733620747336 (+1 ms)Closed at 1733620747336 2024-12-08T01:19:07,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741939_1119 (size=93) 2024-12-08T01:19:07,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741939_1119 (size=93) 2024-12-08T01:19:07,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741939_1119 (size=93) 2024-12-08T01:19:07,341 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:07,341 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620747181) 2024-12-08T01:19:07,355 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=451 (was 444) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2039295362_22 at /127.0.0.1:41288 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:46839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:46839 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2039295362_22 at /127.0.0.1:37844 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2039295362_22 at /127.0.0.1:36348 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=237 (was 249), ProcessCount=11 (was 11), AvailableMemoryMB=9974 (was 10001) 2024-12-08T01:19:07,355 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-08T01:19:07,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T01:19:07,356 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:19:07,356 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:07,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:07,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:07,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:19:07,357 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T01:19:07,357 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=655895052, stopped=false 2024-12-08T01:19:07,357 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=96982154c400,33573,1733620709230 2024-12-08T01:19:07,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:07,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:07,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:07,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:07,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:07,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:07,421 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T01:19:07,421 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:19:07,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:07,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:07,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:07,422 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '96982154c400,41905,1733620710058' ***** 2024-12-08T01:19:07,422 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T01:19:07,422 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '96982154c400,36455,1733620710167' ***** 2024-12-08T01:19:07,422 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T01:19:07,423 INFO [RS:0;96982154c400:41905 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T01:19:07,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:07,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:07,423 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T01:19:07,423 INFO [RS:1;96982154c400:36455 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T01:19:07,423 INFO [RS:1;96982154c400:36455 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T01:19:07,424 INFO [RS:1;96982154c400:36455 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T01:19:07,424 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(959): stopping server 96982154c400,36455,1733620710167 2024-12-08T01:19:07,424 INFO [RS:1;96982154c400:36455 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:19:07,424 INFO [RS:1;96982154c400:36455 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;96982154c400:36455. 2024-12-08T01:19:07,424 DEBUG [RS:1;96982154c400:36455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:07,424 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T01:19:07,424 DEBUG [RS:1;96982154c400:36455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:07,424 INFO [RS:1;96982154c400:36455 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T01:19:07,425 INFO [RS:0;96982154c400:41905 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T01:19:07,425 INFO [RS:1;96982154c400:36455 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T01:19:07,425 INFO [RS:1;96982154c400:36455 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T01:19:07,425 INFO [RS:0;96982154c400:41905 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T01:19:07,425 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T01:19:07,425 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(3091): Received CLOSE for f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:19:07,425 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(959): stopping server 96982154c400,41905,1733620710058 2024-12-08T01:19:07,425 INFO [RS:0;96982154c400:41905 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:19:07,425 INFO [RS:0;96982154c400:41905 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;96982154c400:41905. 2024-12-08T01:19:07,425 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f837f401ea75f84ac401a4defcd14e10, disabling compactions & flushes 2024-12-08T01:19:07,425 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T01:19:07,426 DEBUG [RS:0;96982154c400:41905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:07,426 DEBUG [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T01:19:07,426 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:19:07,426 DEBUG [RS:0;96982154c400:41905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:07,426 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:19:07,426 DEBUG [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:19:07,426 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T01:19:07,426 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. after waiting 0 ms 2024-12-08T01:19:07,426 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T01:19:07,426 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T01:19:07,426 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:19:07,426 DEBUG [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1325): Online Regions={f837f401ea75f84ac401a4defcd14e10=testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10.} 2024-12-08T01:19:07,426 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T01:19:07,426 DEBUG [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1351): Waiting on f837f401ea75f84ac401a4defcd14e10 2024-12-08T01:19:07,426 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T01:19:07,426 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T01:19:07,427 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-08T01:19:07,431 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f837f401ea75f84ac401a4defcd14e10/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-08T01:19:07,432 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/info/f64e0c7b8a594c04a765fe68e141b2ee is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10./info:regioninfo/1733620727895/Put/seqid=0 2024-12-08T01:19:07,432 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:19:07,432 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f837f401ea75f84ac401a4defcd14e10: Waiting for close lock at 1733620747425Running coprocessor pre-close hooks at 1733620747425Disabling compacts and flushes for region at 1733620747425Disabling writes for close at 1733620747426 (+1 ms)Writing region close event to WAL at 1733620747427 (+1 ms)Running coprocessor post-close hooks at 1733620747432 (+5 ms)Closed at 1733620747432 2024-12-08T01:19:07,433 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620723918.f837f401ea75f84ac401a4defcd14e10. 2024-12-08T01:19:07,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741943_1123 (size=6778) 2024-12-08T01:19:07,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741943_1123 (size=6778) 2024-12-08T01:19:07,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741943_1123 (size=6778) 2024-12-08T01:19:07,442 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/info/f64e0c7b8a594c04a765fe68e141b2ee 2024-12-08T01:19:07,449 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/.tmp/info/f64e0c7b8a594c04a765fe68e141b2ee as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/info/f64e0c7b8a594c04a765fe68e141b2ee 2024-12-08T01:19:07,455 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/info/f64e0c7b8a594c04a765fe68e141b2ee, entries=8, sequenceid=23, filesize=6.6 K 2024-12-08T01:19:07,456 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=23, compaction requested=false 2024-12-08T01:19:07,456 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T01:19:07,462 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-08T01:19:07,462 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T01:19:07,462 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T01:19:07,462 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733620747426Running coprocessor pre-close hooks at 1733620747426Disabling compacts and flushes for region at 1733620747426Disabling writes for close at 1733620747426Obtaining lock to block concurrent updates at 1733620747427 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733620747427Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733620747427Flushing stores of hbase:meta,,1.1588230740 at 1733620747428 (+1 ms)Flushing 1588230740/info: creating writer at 1733620747428Flushing 1588230740/info: appending metadata at 1733620747432 (+4 ms)Flushing 1588230740/info: closing flushed file at 1733620747432Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a9a9721: reopening flushed file at 1733620747448 (+16 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=23, compaction requested=false at 1733620747456 (+8 ms)Writing region close event to WAL at 1733620747458 (+2 ms)Running coprocessor post-close hooks at 1733620747462 (+4 ms)Closed at 1733620747462 2024-12-08T01:19:07,462 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T01:19:07,501 INFO [regionserver/96982154c400:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:07,501 INFO [regionserver/96982154c400:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:07,551 INFO [regionserver/96982154c400:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T01:19:07,552 INFO [regionserver/96982154c400:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T01:19:07,626 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(976): stopping server 96982154c400,36455,1733620710167; all regions closed. 2024-12-08T01:19:07,626 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(976): stopping server 96982154c400,41905,1733620710058; all regions closed. 2024-12-08T01:19:07,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741893_1071 (size=2422) 2024-12-08T01:19:07,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741893_1071 (size=2422) 2024-12-08T01:19:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741834_1010 (size=3433) 2024-12-08T01:19:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741893_1071 (size=2422) 2024-12-08T01:19:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741834_1010 (size=3433) 2024-12-08T01:19:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741834_1010 (size=3433) 2024-12-08T01:19:07,636 DEBUG [RS:0;96982154c400:41905 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs 2024-12-08T01:19:07,636 INFO [RS:0;96982154c400:41905 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 96982154c400%2C41905%2C1733620710058:(num 1733620711713) 2024-12-08T01:19:07,636 DEBUG [RS:0;96982154c400:41905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:07,636 DEBUG [RS:1;96982154c400:36455 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs 2024-12-08T01:19:07,636 INFO [RS:0;96982154c400:41905 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:07,636 INFO [RS:1;96982154c400:36455 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 96982154c400%2C36455%2C1733620710167.meta:.meta(num 1733620727061) 2024-12-08T01:19:07,636 INFO [RS:0;96982154c400:41905 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:19:07,636 INFO [RS:0;96982154c400:41905 {}] hbase.ChoreService(370): Chore service for: regionserver/96982154c400:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T01:19:07,636 INFO [RS:0;96982154c400:41905 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T01:19:07,636 INFO [RS:0;96982154c400:41905 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T01:19:07,636 INFO [regionserver/96982154c400:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:19:07,636 INFO [RS:0;96982154c400:41905 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T01:19:07,636 INFO [RS:0;96982154c400:41905 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:19:07,637 INFO [RS:0;96982154c400:41905 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41905 2024-12-08T01:19:07,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741833_1009 (size=93) 2024-12-08T01:19:07,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741833_1009 (size=93) 2024-12-08T01:19:07,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741833_1009 (size=93) 2024-12-08T01:19:07,640 DEBUG [RS:1;96982154c400:36455 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/oldWALs 2024-12-08T01:19:07,641 INFO [RS:1;96982154c400:36455 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 96982154c400%2C36455%2C1733620710167:(num 1733620711711) 2024-12-08T01:19:07,641 DEBUG [RS:1;96982154c400:36455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:07,641 INFO [RS:1;96982154c400:36455 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:07,641 INFO [RS:1;96982154c400:36455 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:19:07,641 INFO [RS:1;96982154c400:36455 {}] hbase.ChoreService(370): Chore service for: regionserver/96982154c400:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T01:19:07,641 INFO [RS:1;96982154c400:36455 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:19:07,641 INFO [regionserver/96982154c400:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:19:07,641 INFO [RS:1;96982154c400:36455 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36455 2024-12-08T01:19:07,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/96982154c400,41905,1733620710058 2024-12-08T01:19:07,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:19:07,662 INFO [RS:0;96982154c400:41905 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:19:07,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/96982154c400,36455,1733620710167 2024-12-08T01:19:07,674 INFO [RS:1;96982154c400:36455 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:19:07,685 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [96982154c400,36455,1733620710167] 2024-12-08T01:19:07,706 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/96982154c400,36455,1733620710167 already deleted, retry=false 2024-12-08T01:19:07,706 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 96982154c400,36455,1733620710167 expired; onlineServers=1 2024-12-08T01:19:07,707 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [96982154c400,41905,1733620710058] 2024-12-08T01:19:07,716 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/96982154c400,41905,1733620710058 already deleted, retry=false 2024-12-08T01:19:07,716 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 96982154c400,41905,1733620710058 expired; onlineServers=0 2024-12-08T01:19:07,717 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '96982154c400,33573,1733620709230' ***** 2024-12-08T01:19:07,717 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T01:19:07,717 INFO [M:0;96982154c400:33573 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:19:07,717 INFO [M:0;96982154c400:33573 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:19:07,717 DEBUG [M:0;96982154c400:33573 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T01:19:07,717 DEBUG [M:0;96982154c400:33573 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T01:19:07,717 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T01:19:07,717 DEBUG [master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620711334 {}] cleaner.HFileCleaner(306): Exit Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620711334,5,FailOnTimeoutGroup] 2024-12-08T01:19:07,717 DEBUG [master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620711338 {}] cleaner.HFileCleaner(306): Exit Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620711338,5,FailOnTimeoutGroup] 2024-12-08T01:19:07,718 INFO [M:0;96982154c400:33573 {}] hbase.ChoreService(370): Chore service for: master/96982154c400:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T01:19:07,718 INFO [M:0;96982154c400:33573 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:19:07,718 DEBUG [M:0;96982154c400:33573 {}] master.HMaster(1795): Stopping service threads 2024-12-08T01:19:07,718 INFO [M:0;96982154c400:33573 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T01:19:07,719 INFO [M:0;96982154c400:33573 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T01:19:07,719 INFO [M:0;96982154c400:33573 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T01:19:07,720 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T01:19:07,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T01:19:07,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:07,727 DEBUG [M:0;96982154c400:33573 {}] zookeeper.ZKUtil(347): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T01:19:07,727 WARN [M:0;96982154c400:33573 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T01:19:07,728 INFO [M:0;96982154c400:33573 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/.lastflushedseqids 2024-12-08T01:19:07,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741944_1124 (size=119) 2024-12-08T01:19:07,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741944_1124 (size=119) 2024-12-08T01:19:07,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741944_1124 (size=119) 2024-12-08T01:19:07,745 INFO [M:0;96982154c400:33573 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T01:19:07,745 INFO [M:0;96982154c400:33573 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T01:19:07,745 DEBUG [M:0;96982154c400:33573 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T01:19:07,745 INFO [M:0;96982154c400:33573 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:07,745 DEBUG [M:0;96982154c400:33573 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:07,745 DEBUG [M:0;96982154c400:33573 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T01:19:07,745 DEBUG [M:0;96982154c400:33573 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:07,745 INFO [M:0;96982154c400:33573 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-12-08T01:19:07,762 DEBUG [M:0;96982154c400:33573 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/84a424f6bc944597807fcb7792eeac58 is 82, key is hbase:meta,,1/info:regioninfo/1733620727203/Put/seqid=0 2024-12-08T01:19:07,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741945_1125 (size=6063) 2024-12-08T01:19:07,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741945_1125 (size=6063) 2024-12-08T01:19:07,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741945_1125 (size=6063) 2024-12-08T01:19:07,769 INFO [M:0;96982154c400:33573 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/84a424f6bc944597807fcb7792eeac58 2024-12-08T01:19:07,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:07,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41905-0x1000300bb2c0001, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:07,785 INFO [RS:0;96982154c400:41905 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:19:07,785 INFO [RS:0;96982154c400:41905 {}] regionserver.HRegionServer(1031): Exiting; stopping=96982154c400,41905,1733620710058; zookeeper connection closed. 2024-12-08T01:19:07,785 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1c4e9616 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1c4e9616 2024-12-08T01:19:07,786 DEBUG [M:0;96982154c400:33573 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88f039a362eb48bf8c0c5f0b82681f42 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733620724358/Put/seqid=0 2024-12-08T01:19:07,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741946_1126 (size=7906) 2024-12-08T01:19:07,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741946_1126 (size=7906) 2024-12-08T01:19:07,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741946_1126 (size=7906) 2024-12-08T01:19:07,794 INFO [M:0;96982154c400:33573 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88f039a362eb48bf8c0c5f0b82681f42 2024-12-08T01:19:07,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:07,796 INFO [RS:1;96982154c400:36455 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:19:07,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36455-0x1000300bb2c0002, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:07,796 INFO [RS:1;96982154c400:36455 {}] regionserver.HRegionServer(1031): Exiting; stopping=96982154c400,36455,1733620710167; zookeeper connection closed. 2024-12-08T01:19:07,796 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@26209594 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@26209594 2024-12-08T01:19:07,796 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-08T01:19:07,798 INFO [M:0;96982154c400:33573 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 88f039a362eb48bf8c0c5f0b82681f42 2024-12-08T01:19:07,811 DEBUG [M:0;96982154c400:33573 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf0ac8a26a2e4bc5b29c508f84095807 is 69, key is 96982154c400,36455,1733620710167/rs:state/1733620711425/Put/seqid=0 2024-12-08T01:19:07,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741947_1127 (size=5440) 2024-12-08T01:19:07,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741947_1127 (size=5440) 2024-12-08T01:19:07,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741947_1127 (size=5440) 2024-12-08T01:19:07,818 INFO [M:0;96982154c400:33573 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf0ac8a26a2e4bc5b29c508f84095807 2024-12-08T01:19:07,823 INFO [M:0;96982154c400:33573 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf0ac8a26a2e4bc5b29c508f84095807 2024-12-08T01:19:07,824 DEBUG [M:0;96982154c400:33573 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/84a424f6bc944597807fcb7792eeac58 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/84a424f6bc944597807fcb7792eeac58 2024-12-08T01:19:07,830 INFO [M:0;96982154c400:33573 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/84a424f6bc944597807fcb7792eeac58, entries=14, sequenceid=207, filesize=5.9 K 2024-12-08T01:19:07,831 DEBUG [M:0;96982154c400:33573 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88f039a362eb48bf8c0c5f0b82681f42 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/88f039a362eb48bf8c0c5f0b82681f42 2024-12-08T01:19:07,836 INFO [M:0;96982154c400:33573 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 88f039a362eb48bf8c0c5f0b82681f42 2024-12-08T01:19:07,836 INFO [M:0;96982154c400:33573 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/88f039a362eb48bf8c0c5f0b82681f42, entries=21, sequenceid=207, filesize=7.7 K 2024-12-08T01:19:07,837 DEBUG [M:0;96982154c400:33573 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf0ac8a26a2e4bc5b29c508f84095807 as hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cf0ac8a26a2e4bc5b29c508f84095807 2024-12-08T01:19:07,842 INFO [M:0;96982154c400:33573 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf0ac8a26a2e4bc5b29c508f84095807 2024-12-08T01:19:07,842 INFO [M:0;96982154c400:33573 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/f2842151-3718-7ee4-9f30-dcbb6d92fcfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cf0ac8a26a2e4bc5b29c508f84095807, entries=3, sequenceid=207, filesize=5.3 K 2024-12-08T01:19:07,843 INFO [M:0;96982154c400:33573 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 98ms, sequenceid=207, compaction requested=false 2024-12-08T01:19:07,844 INFO [M:0;96982154c400:33573 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:07,844 DEBUG [M:0;96982154c400:33573 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733620747745Disabling compacts and flushes for region at 1733620747745Disabling writes for close at 1733620747745Obtaining lock to block concurrent updates at 1733620747745Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733620747745Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1733620747746 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733620747746Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733620747746Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733620747762 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733620747762Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733620747773 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733620747786 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733620747786Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733620747798 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733620747810 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733620747810Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12d1274a: reopening flushed file at 1733620747823 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d997f92: reopening flushed file at 1733620747830 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6565551d: reopening flushed file at 1733620747836 (+6 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 98ms, sequenceid=207, compaction requested=false at 1733620747843 (+7 ms)Writing region close event to WAL at 1733620747844 (+1 ms)Closed at 1733620747844 2024-12-08T01:19:07,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741830_1006 (size=99424) 2024-12-08T01:19:07,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45653 is added to blk_1073741830_1006 (size=99424) 2024-12-08T01:19:07,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46125 is added to blk_1073741830_1006 (size=99424) 2024-12-08T01:19:07,847 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:19:07,847 INFO [M:0;96982154c400:33573 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T01:19:07,847 INFO [M:0;96982154c400:33573 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33573 2024-12-08T01:19:07,847 INFO [M:0;96982154c400:33573 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:19:07,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:07,953 INFO [M:0;96982154c400:33573 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:19:07,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33573-0x1000300bb2c0000, quorum=127.0.0.1:52656, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:07,964 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620737603 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620737603 (inode 16688) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620728389/wal.1733620737603 (inode 16688) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:07,967 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728261 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728261 (inode 16665) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620727976/wal.1733620728261 (inode 16665) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:07,968 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620719429/wal.1733620719636 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:07,970 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620738276 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620738276 (inode 16714) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620737778/wal.1733620738276 (inode 16714) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:07,970 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620738436/wal.1733620738493 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:07,971 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620742833/wal.1733620742994 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:07,971 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620713268/wal.1733620713334 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:07,973 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal.1733620713142 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal.1733620713142 (inode 16485) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620712982/wal.1733620713142 (inode 16485) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:07,975 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal.1733620712758 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal.1733620712758 (inode 16462) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620712536/wal.1733620712758 (inode 16462) Holder DFSClient_NONMAPREDUCE_1774669608_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:07,980 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:07,982 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:19:07,983 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:19:07,983 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:19:07,983 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir/,STOPPED} 2024-12-08T01:19:07,986 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T01:19:07,986 WARN [BP-1537938077-172.17.0.2-1733620704924 heartbeating to localhost/127.0.0.1:46839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T01:19:07,986 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T01:19:07,986 WARN [BP-1537938077-172.17.0.2-1733620704924 heartbeating to localhost/127.0.0.1:46839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1537938077-172.17.0.2-1733620704924 (Datanode Uuid d35e6347-9819-45b6-b8d9-5ac18385aaf9) service to localhost/127.0.0.1:46839 2024-12-08T01:19:07,987 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data5/current/BP-1537938077-172.17.0.2-1733620704924 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:07,988 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data6/current/BP-1537938077-172.17.0.2-1733620704924 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:07,988 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T01:19:07,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:07,990 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:19:07,990 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:19:07,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:19:07,991 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir/,STOPPED} 2024-12-08T01:19:07,991 WARN [BP-1537938077-172.17.0.2-1733620704924 heartbeating to localhost/127.0.0.1:46839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T01:19:07,991 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T01:19:07,991 WARN [BP-1537938077-172.17.0.2-1733620704924 heartbeating to localhost/127.0.0.1:46839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1537938077-172.17.0.2-1733620704924 (Datanode Uuid 9f7aa09e-f157-41b4-8217-8a9b0a69ef02) service to localhost/127.0.0.1:46839 2024-12-08T01:19:07,991 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T01:19:07,992 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data3/current/BP-1537938077-172.17.0.2-1733620704924 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:07,992 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data4/current/BP-1537938077-172.17.0.2-1733620704924 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:07,992 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T01:19:07,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:07,995 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:19:07,995 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:19:07,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:19:07,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir/,STOPPED} 2024-12-08T01:19:07,996 WARN [BP-1537938077-172.17.0.2-1733620704924 heartbeating to localhost/127.0.0.1:46839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T01:19:07,996 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T01:19:07,996 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T01:19:07,996 WARN [BP-1537938077-172.17.0.2-1733620704924 heartbeating to localhost/127.0.0.1:46839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1537938077-172.17.0.2-1733620704924 (Datanode Uuid d4b0216d-adfc-443e-acf6-d06cad823a5c) service to localhost/127.0.0.1:46839 2024-12-08T01:19:07,996 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data1/current/BP-1537938077-172.17.0.2-1733620704924 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:07,997 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/cluster_19f99da8-e534-d042-2e21-a67c06099ac8/data/data2/current/BP-1537938077-172.17.0.2-1733620704924 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:07,997 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T01:19:08,002 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T01:19:08,003 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:19:08,003 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:19:08,003 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:19:08,003 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b7585a93-5ee7-996a-02c0-fbae63347b8f/hadoop.log.dir/,STOPPED} 2024-12-08T01:19:08,013 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T01:19:08,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down