2024-11-24 09:40:24,891 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 09:40:24,913 main DEBUG Took 0.015661 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 09:40:24,914 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 09:40:24,914 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 09:40:24,916 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 09:40:24,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,930 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 09:40:24,967 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,970 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,971 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,972 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,974 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,982 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,987 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,988 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,989 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,990 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,991 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,992 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:40:24,994 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:24,994 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 09:40:24,996 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:40:25,009 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 09:40:25,012 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 09:40:25,012 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 09:40:25,020 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 09:40:25,021 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 09:40:25,062 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 09:40:25,067 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 09:40:25,074 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 09:40:25,075 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 09:40:25,076 main DEBUG createAppenders(={Console}) 2024-11-24 09:40:25,077 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-24 09:40:25,077 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 09:40:25,078 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-24 09:40:25,078 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 09:40:25,079 main DEBUG OutputStream closed 2024-11-24 09:40:25,079 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 09:40:25,079 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 09:40:25,080 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-24 09:40:25,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 09:40:25,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 09:40:25,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 09:40:25,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 09:40:25,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 09:40:25,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 09:40:25,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 09:40:25,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 09:40:25,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 09:40:25,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 09:40:25,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 09:40:25,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 09:40:25,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 09:40:25,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 09:40:25,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 09:40:25,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 09:40:25,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 09:40:25,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 09:40:25,284 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 09:40:25,285 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-24 09:40:25,285 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 09:40:25,286 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-24T09:40:25,660 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4 2024-11-24 09:40:25,666 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 09:40:25,667 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T09:40:25,678 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T09:40:25,691 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-11-24T09:40:25,722 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T09:40:25,783 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T09:40:25,783 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T09:40:25,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T09:40:25,830 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d, deleteOnExit=true 2024-11-24T09:40:25,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T09:40:25,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/test.cache.data in system properties and HBase conf 2024-11-24T09:40:25,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T09:40:25,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir in system properties and HBase conf 2024-11-24T09:40:25,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T09:40:25,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T09:40:25,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T09:40:25,942 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T09:40:26,109 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:40:26,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T09:40:26,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T09:40:26,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T09:40:26,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T09:40:26,129 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T09:40:26,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T09:40:26,133 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T09:40:26,134 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T09:40:26,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T09:40:26,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/nfs.dump.dir in system properties and HBase conf 2024-11-24T09:40:26,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/java.io.tmpdir in system properties and HBase conf 2024-11-24T09:40:26,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T09:40:26,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T09:40:26,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T09:40:27,479 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T09:40:27,598 INFO [Time-limited test {}] log.Log(170): Logging initialized @3868ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T09:40:27,692 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:40:27,782 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T09:40:27,864 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T09:40:27,864 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T09:40:27,867 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T09:40:27,890 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:40:27,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir/,AVAILABLE} 2024-11-24T09:40:27,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T09:40:28,239 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/java.io.tmpdir/jetty-localhost-45477-hadoop-hdfs-3_4_1-tests_jar-_-any-12338324067024533530/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T09:40:28,263 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:45477} 2024-11-24T09:40:28,264 INFO [Time-limited test {}] server.Server(415): Started @4534ms 2024-11-24T09:40:29,034 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:40:29,043 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T09:40:29,044 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T09:40:29,045 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T09:40:29,045 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T09:40:29,046 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7bbda1ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir/,AVAILABLE} 2024-11-24T09:40:29,047 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55114173{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T09:40:29,181 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cdc8afb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/java.io.tmpdir/jetty-localhost-44569-hadoop-hdfs-3_4_1-tests_jar-_-any-14960062824210747388/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:40:29,182 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@796dc018{HTTP/1.1, (http/1.1)}{localhost:44569} 2024-11-24T09:40:29,182 INFO [Time-limited test {}] server.Server(415): Started @5452ms 2024-11-24T09:40:29,249 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T09:40:29,403 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:40:29,412 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T09:40:29,416 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T09:40:29,417 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T09:40:29,417 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T09:40:29,425 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b29fce4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir/,AVAILABLE} 2024-11-24T09:40:29,431 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f53d973{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T09:40:29,577 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42bdeb7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/java.io.tmpdir/jetty-localhost-36949-hadoop-hdfs-3_4_1-tests_jar-_-any-5851399520171881044/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:40:29,578 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@486e75d4{HTTP/1.1, (http/1.1)}{localhost:36949} 2024-11-24T09:40:29,581 INFO [Time-limited test {}] server.Server(415): Started @5851ms 2024-11-24T09:40:29,584 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T09:40:29,679 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T09:40:29,689 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T09:40:29,723 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T09:40:29,723 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T09:40:29,723 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T09:40:29,727 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@137d43c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir/,AVAILABLE} 2024-11-24T09:40:29,728 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7701b046{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T09:40:29,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@11039a9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/java.io.tmpdir/jetty-localhost-39991-hadoop-hdfs-3_4_1-tests_jar-_-any-10608866888138257459/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:40:29,871 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7000b5ce{HTTP/1.1, (http/1.1)}{localhost:39991} 2024-11-24T09:40:29,872 INFO [Time-limited test {}] server.Server(415): Started @6142ms 2024-11-24T09:40:29,874 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T09:40:30,678 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data3/current/BP-321562900-172.17.0.2-1732441227025/current, will proceed with Du for space computation calculation, 2024-11-24T09:40:30,678 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data2/current/BP-321562900-172.17.0.2-1732441227025/current, will proceed with Du for space computation calculation, 2024-11-24T09:40:30,682 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data1/current/BP-321562900-172.17.0.2-1732441227025/current, will proceed with Du for space computation calculation, 2024-11-24T09:40:30,683 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data4/current/BP-321562900-172.17.0.2-1732441227025/current, will proceed with Du for space computation calculation, 2024-11-24T09:40:30,756 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T09:40:30,766 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T09:40:30,799 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data5/current/BP-321562900-172.17.0.2-1732441227025/current, will proceed with Du for space computation calculation, 2024-11-24T09:40:30,800 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data6/current/BP-321562900-172.17.0.2-1732441227025/current, will proceed with Du for space computation calculation, 2024-11-24T09:40:30,845 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe252fe5f9ae86ed2 with lease ID 0xb9c05f73bcf75c3a: Processing first storage report for DS-6e28db07-2977-48d6-98a2-40409d0bf5d3 from datanode DatanodeRegistration(127.0.0.1:43599, datanodeUuid=73904766-b6c2-4e27-9693-9a1e641ebc27, infoPort=34113, infoSecurePort=0, ipcPort=41433, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025) 2024-11-24T09:40:30,846 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe252fe5f9ae86ed2 with lease ID 0xb9c05f73bcf75c3a: from storage DS-6e28db07-2977-48d6-98a2-40409d0bf5d3 node DatanodeRegistration(127.0.0.1:43599, datanodeUuid=73904766-b6c2-4e27-9693-9a1e641ebc27, infoPort=34113, infoSecurePort=0, ipcPort=41433, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-24T09:40:30,848 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b7acd991a479df with lease ID 0xb9c05f73bcf75c3b: Processing first storage report for DS-c38566f6-ad2a-4a7a-83c5-fff68116226e from datanode DatanodeRegistration(127.0.0.1:45495, datanodeUuid=78e58b0a-1a42-4227-bab3-490aa0a195c5, infoPort=39043, infoSecurePort=0, ipcPort=36399, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025) 2024-11-24T09:40:30,848 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b7acd991a479df with lease ID 0xb9c05f73bcf75c3b: from storage DS-c38566f6-ad2a-4a7a-83c5-fff68116226e node DatanodeRegistration(127.0.0.1:45495, datanodeUuid=78e58b0a-1a42-4227-bab3-490aa0a195c5, infoPort=39043, infoSecurePort=0, ipcPort=36399, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T09:40:30,848 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe252fe5f9ae86ed2 with lease ID 0xb9c05f73bcf75c3a: Processing first storage report for DS-35359805-69fb-4a12-848f-15825846edfe from datanode DatanodeRegistration(127.0.0.1:43599, datanodeUuid=73904766-b6c2-4e27-9693-9a1e641ebc27, infoPort=34113, infoSecurePort=0, ipcPort=41433, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025) 2024-11-24T09:40:30,849 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe252fe5f9ae86ed2 with lease ID 0xb9c05f73bcf75c3a: from storage DS-35359805-69fb-4a12-848f-15825846edfe node DatanodeRegistration(127.0.0.1:43599, datanodeUuid=73904766-b6c2-4e27-9693-9a1e641ebc27, infoPort=34113, infoSecurePort=0, ipcPort=41433, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T09:40:30,856 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b7acd991a479df with lease ID 0xb9c05f73bcf75c3b: Processing first storage report for DS-9e404ddb-eaf0-4c9e-81d0-7a9bf78955c9 from datanode DatanodeRegistration(127.0.0.1:45495, datanodeUuid=78e58b0a-1a42-4227-bab3-490aa0a195c5, infoPort=39043, infoSecurePort=0, ipcPort=36399, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025) 2024-11-24T09:40:30,856 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b7acd991a479df with lease ID 0xb9c05f73bcf75c3b: from storage DS-9e404ddb-eaf0-4c9e-81d0-7a9bf78955c9 node DatanodeRegistration(127.0.0.1:45495, datanodeUuid=78e58b0a-1a42-4227-bab3-490aa0a195c5, infoPort=39043, infoSecurePort=0, ipcPort=36399, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T09:40:30,913 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T09:40:30,922 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xef2207d78f8a8e7a with lease ID 0xb9c05f73bcf75c3c: Processing first storage report for DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29 from datanode DatanodeRegistration(127.0.0.1:45595, datanodeUuid=ce27ccbc-32d8-41ca-afbc-75575b160ac7, infoPort=33173, infoSecurePort=0, ipcPort=39539, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025) 2024-11-24T09:40:30,923 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef2207d78f8a8e7a with lease ID 0xb9c05f73bcf75c3c: from storage DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29 node DatanodeRegistration(127.0.0.1:45595, datanodeUuid=ce27ccbc-32d8-41ca-afbc-75575b160ac7, infoPort=33173, infoSecurePort=0, ipcPort=39539, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T09:40:30,923 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xef2207d78f8a8e7a with lease ID 0xb9c05f73bcf75c3c: Processing first storage report for DS-33e944e0-2d10-472a-b183-5127eaadd432 from datanode DatanodeRegistration(127.0.0.1:45595, datanodeUuid=ce27ccbc-32d8-41ca-afbc-75575b160ac7, infoPort=33173, infoSecurePort=0, ipcPort=39539, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025) 2024-11-24T09:40:30,923 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef2207d78f8a8e7a with lease ID 0xb9c05f73bcf75c3c: from storage DS-33e944e0-2d10-472a-b183-5127eaadd432 node DatanodeRegistration(127.0.0.1:45595, datanodeUuid=ce27ccbc-32d8-41ca-afbc-75575b160ac7, infoPort=33173, infoSecurePort=0, ipcPort=39539, storageInfo=lv=-57;cid=testClusterID;nsid=2045928316;c=1732441227025), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T09:40:30,941 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4 2024-11-24T09:40:31,036 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/zookeeper_0, clientPort=52387, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T09:40:31,052 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52387 2024-11-24T09:40:31,066 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:31,070 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:31,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741825_1001 (size=7) 2024-11-24T09:40:31,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741825_1001 (size=7) 2024-11-24T09:40:31,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741825_1001 (size=7) 2024-11-24T09:40:31,821 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63 with version=8 2024-11-24T09:40:31,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/hbase-staging 2024-11-24T09:40:32,177 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:40:32,188 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:32,189 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:32,195 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:40:32,196 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:32,196 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:40:32,364 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T09:40:32,431 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T09:40:32,442 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T09:40:32,446 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:40:32,476 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3475 (auto-detected) 2024-11-24T09:40:32,477 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T09:40:32,509 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43385 2024-11-24T09:40:32,541 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43385 connecting to ZooKeeper ensemble=127.0.0.1:52387 2024-11-24T09:40:32,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433850x0, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:40:32,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43385-0x1016d80e1650000 connected 2024-11-24T09:40:32,829 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:32,835 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:32,864 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:40:32,874 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63, hbase.cluster.distributed=false 2024-11-24T09:40:32,949 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:40:32,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43385 2024-11-24T09:40:32,983 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43385 2024-11-24T09:40:32,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43385 2024-11-24T09:40:32,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43385 2024-11-24T09:40:33,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43385 2024-11-24T09:40:33,189 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:40:33,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,191 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:40:33,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:40:33,195 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T09:40:33,199 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:40:33,200 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36811 2024-11-24T09:40:33,202 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36811 connecting to ZooKeeper ensemble=127.0.0.1:52387 2024-11-24T09:40:33,204 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:33,208 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368110x0, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:40:33,268 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:368110x0, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:40:33,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36811-0x1016d80e1650001 connected 2024-11-24T09:40:33,272 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T09:40:33,281 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T09:40:33,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T09:40:33,291 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:40:33,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36811 2024-11-24T09:40:33,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36811 2024-11-24T09:40:33,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36811 2024-11-24T09:40:33,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36811 2024-11-24T09:40:33,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36811 2024-11-24T09:40:33,325 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:40:33,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,327 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:40:33,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:40:33,327 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T09:40:33,327 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:40:33,329 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42723 2024-11-24T09:40:33,331 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42723 connecting to ZooKeeper ensemble=127.0.0.1:52387 2024-11-24T09:40:33,333 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:33,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:33,360 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427230x0, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:40:33,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427230x0, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:40:33,360 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T09:40:33,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42723-0x1016d80e1650002 connected 2024-11-24T09:40:33,361 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T09:40:33,362 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T09:40:33,364 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:40:33,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42723 2024-11-24T09:40:33,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42723 2024-11-24T09:40:33,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42723 2024-11-24T09:40:33,367 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42723 2024-11-24T09:40:33,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42723 2024-11-24T09:40:33,385 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:40:33,385 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,386 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:40:33,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:40:33,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:40:33,386 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T09:40:33,386 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:40:33,387 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35917 2024-11-24T09:40:33,389 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35917 connecting to ZooKeeper ensemble=127.0.0.1:52387 2024-11-24T09:40:33,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:33,393 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:33,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359170x0, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:40:33,448 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T09:40:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359170x0, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:40:33,463 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T09:40:33,465 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35917-0x1016d80e1650003 connected 2024-11-24T09:40:33,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T09:40:33,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:40:33,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35917 2024-11-24T09:40:33,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35917 2024-11-24T09:40:33,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35917 2024-11-24T09:40:33,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35917 2024-11-24T09:40:33,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35917 2024-11-24T09:40:33,517 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba35ada4783f:43385 2024-11-24T09:40:33,518 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba35ada4783f,43385,1732441231965 2024-11-24T09:40:33,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:40:33,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:40:33,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:40:33,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:40:33,544 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba35ada4783f,43385,1732441231965 2024-11-24T09:40:33,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T09:40:33,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T09:40:33,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T09:40:33,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:33,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:33,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:33,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:33,578 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T09:40:33,580 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba35ada4783f,43385,1732441231965 from backup master directory 2024-11-24T09:40:33,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:40:33,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:40:33,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:40:33,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba35ada4783f,43385,1732441231965 2024-11-24T09:40:33,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:40:33,595 WARN [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:40:33,595 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba35ada4783f,43385,1732441231965 2024-11-24T09:40:33,597 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T09:40:33,599 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T09:40:33,659 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/hbase.id] with ID: f30c0795-04ef-4a63-a29c-8fba537cfc7d 2024-11-24T09:40:33,659 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/.tmp/hbase.id 2024-11-24T09:40:33,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741826_1002 (size=42) 2024-11-24T09:40:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741826_1002 (size=42) 2024-11-24T09:40:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741826_1002 (size=42) 2024-11-24T09:40:33,679 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/.tmp/hbase.id]:[hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/hbase.id] 2024-11-24T09:40:33,734 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T09:40:33,744 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T09:40:33,767 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-24T09:40:33,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:33,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:33,802 WARN [IPC Server handler 4 on default port 43013 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-24T09:40:33,803 WARN [IPC Server handler 4 on default port 43013 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-24T09:40:33,803 WARN [IPC Server handler 4 on default port 43013 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-24T09:40:33,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741827_1003 (size=196) 2024-11-24T09:40:33,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741827_1003 (size=196) 2024-11-24T09:40:33,831 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T09:40:33,834 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T09:40:33,841 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:33,880 WARN [IPC Server handler 0 on default port 43013 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-24T09:40:33,880 WARN [IPC Server handler 0 on default port 43013 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-24T09:40:33,881 WARN [IPC Server handler 0 on default port 43013 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-24T09:40:33,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741828_1004 (size=1189) 2024-11-24T09:40:33,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741828_1004 (size=1189) 2024-11-24T09:40:33,936 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store 2024-11-24T09:40:33,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741829_1005 (size=34) 2024-11-24T09:40:33,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741829_1005 (size=34) 2024-11-24T09:40:33,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741829_1005 (size=34) 2024-11-24T09:40:33,990 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T09:40:33,994 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:33,996 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T09:40:33,996 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:40:33,996 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:40:33,998 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T09:40:33,998 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:40:33,998 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:40:33,999 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732441233995Disabling compacts and flushes for region at 1732441233996 (+1 ms)Disabling writes for close at 1732441233998 (+2 ms)Writing region close event to WAL at 1732441233998Closed at 1732441233998 2024-11-24T09:40:34,002 WARN [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/.initializing 2024-11-24T09:40:34,002 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/WALs/ba35ada4783f,43385,1732441231965 2024-11-24T09:40:34,020 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:34,039 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C43385%2C1732441231965, suffix=, logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/WALs/ba35ada4783f,43385,1732441231965, archiveDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/oldWALs, maxLogs=10 2024-11-24T09:40:34,078 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/WALs/ba35ada4783f,43385,1732441231965/ba35ada4783f%2C43385%2C1732441231965.1732441234044, exclude list is [], retry=0 2024-11-24T09:40:34,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:34,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:34,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:34,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:34,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T09:40:34,162 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/WALs/ba35ada4783f,43385,1732441231965/ba35ada4783f%2C43385%2C1732441231965.1732441234044 2024-11-24T09:40:34,163 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:34,164 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:34,164 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:34,168 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,169 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T09:40:34,252 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:34,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:34,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T09:40:34,268 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:34,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:34,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,275 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T09:40:34,275 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:34,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:34,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,282 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T09:40:34,283 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:34,284 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:34,285 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,289 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,291 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,300 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,301 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,305 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:40:34,313 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:40:34,321 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:34,323 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60651153, jitterRate=-0.09622739255428314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:40:34,333 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732441234184Initializing all the Stores at 1732441234187 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441234188 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441234189 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441234189Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441234189Cleaning up temporary data from old regions at 1732441234301 (+112 ms)Region opened successfully at 1732441234333 (+32 ms) 2024-11-24T09:40:34,335 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T09:40:34,383 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bbed3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:40:34,421 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T09:40:34,435 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T09:40:34,435 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T09:40:34,438 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T09:40:34,440 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T09:40:34,447 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T09:40:34,447 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T09:40:34,479 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T09:40:34,490 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T09:40:34,533 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T09:40:34,537 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T09:40:34,538 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T09:40:34,550 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T09:40:34,553 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T09:40:34,556 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T09:40:34,567 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T09:40:34,568 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T09:40:34,575 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T09:40:34,594 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T09:40:34,600 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T09:40:34,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:40:34,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:40:34,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:40:34,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:40:34,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,613 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba35ada4783f,43385,1732441231965, sessionid=0x1016d80e1650000, setting cluster-up flag (Was=false) 2024-11-24T09:40:34,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,667 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T09:40:34,670 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba35ada4783f,43385,1732441231965 2024-11-24T09:40:34,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:34,717 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T09:40:34,720 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba35ada4783f,43385,1732441231965 2024-11-24T09:40:34,728 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T09:40:34,812 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(746): ClusterId : f30c0795-04ef-4a63-a29c-8fba537cfc7d 2024-11-24T09:40:34,813 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(746): ClusterId : f30c0795-04ef-4a63-a29c-8fba537cfc7d 2024-11-24T09:40:34,814 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T09:40:34,815 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T09:40:34,815 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(746): ClusterId : f30c0795-04ef-4a63-a29c-8fba537cfc7d 2024-11-24T09:40:34,815 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T09:40:34,828 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T09:40:34,838 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T09:40:34,844 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T09:40:34,844 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T09:40:34,844 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T09:40:34,845 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T09:40:34,845 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T09:40:34,845 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T09:40:34,849 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T09:40:34,852 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T09:40:34,852 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T09:40:34,852 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T09:40:34,853 DEBUG [RS:2;ba35ada4783f:35917 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e00fe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:40:34,853 DEBUG [RS:1;ba35ada4783f:42723 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338599b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:40:34,853 DEBUG [RS:0;ba35ada4783f:36811 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f418af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:40:34,858 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba35ada4783f,43385,1732441231965 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T09:40:34,874 DEBUG [RS:1;ba35ada4783f:42723 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ba35ada4783f:42723 2024-11-24T09:40:34,875 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:40:34,875 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:40:34,875 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:40:34,875 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:40:34,876 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba35ada4783f:0, corePoolSize=10, maxPoolSize=10 2024-11-24T09:40:34,876 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba35ada4783f:36811 2024-11-24T09:40:34,876 DEBUG [RS:2;ba35ada4783f:35917 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ba35ada4783f:35917 2024-11-24T09:40:34,878 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T09:40:34,876 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:34,878 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T09:40:34,878 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T09:40:34,878 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T09:40:34,878 DEBUG [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T09:40:34,878 DEBUG [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T09:40:34,878 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:40:34,879 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:34,881 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba35ada4783f,43385,1732441231965 with port=42723, startcode=1732441233325 2024-11-24T09:40:34,882 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba35ada4783f,43385,1732441231965 with port=35917, startcode=1732441233385 2024-11-24T09:40:34,884 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T09:40:34,885 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T09:40:34,885 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T09:40:34,886 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba35ada4783f,43385,1732441231965 with port=36811, startcode=1732441233150 2024-11-24T09:40:34,897 DEBUG [RS:1;ba35ada4783f:42723 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T09:40:34,897 DEBUG [RS:0;ba35ada4783f:36811 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T09:40:34,898 DEBUG [RS:2;ba35ada4783f:35917 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T09:40:34,935 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732441264935 2024-11-24T09:40:34,937 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T09:40:34,938 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T09:40:34,950 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T09:40:34,954 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T09:40:34,955 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T09:40:34,955 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T09:40:34,960 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:40:34,961 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T09:40:34,960 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:34,968 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:34,969 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T09:40:34,972 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T09:40:34,973 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T09:40:34,973 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T09:40:34,975 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T09:40:34,976 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T09:40:34,982 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732441234978,5,FailOnTimeoutGroup] 2024-11-24T09:40:34,985 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42427, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T09:40:34,985 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40737, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T09:40:34,985 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33119, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T09:40:34,991 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732441234984,5,FailOnTimeoutGroup] 2024-11-24T09:40:34,991 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:34,991 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T09:40:35,016 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,016 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,017 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T09:40:35,023 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba35ada4783f,35917,1732441233385 2024-11-24T09:40:35,028 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(517): Registering regionserver=ba35ada4783f,35917,1732441233385 2024-11-24T09:40:35,036 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba35ada4783f,42723,1732441233325 2024-11-24T09:40:35,037 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(517): Registering regionserver=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:35,037 DEBUG [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63 2024-11-24T09:40:35,037 DEBUG [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43013 2024-11-24T09:40:35,037 DEBUG [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T09:40:35,043 DEBUG [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63 2024-11-24T09:40:35,043 DEBUG [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43013 2024-11-24T09:40:35,043 DEBUG [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T09:40:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741831_1007 (size=1321) 2024-11-24T09:40:35,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741831_1007 (size=1321) 2024-11-24T09:40:35,051 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T09:40:35,051 WARN [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T09:40:35,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741831_1007 (size=1321) 2024-11-24T09:40:35,054 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T09:40:35,055 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63 2024-11-24T09:40:35,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:40:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741832_1008 (size=32) 2024-11-24T09:40:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741832_1008 (size=32) 2024-11-24T09:40:35,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741832_1008 (size=32) 2024-11-24T09:40:35,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:35,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T09:40:35,092 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T09:40:35,093 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:35,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:35,094 DEBUG [RS:2;ba35ada4783f:35917 {}] zookeeper.ZKUtil(111): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba35ada4783f,35917,1732441233385 2024-11-24T09:40:35,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T09:40:35,094 WARN [RS:2;ba35ada4783f:35917 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:40:35,094 INFO [RS:2;ba35ada4783f:35917 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:35,095 DEBUG [RS:1;ba35ada4783f:42723 {}] zookeeper.ZKUtil(111): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba35ada4783f,42723,1732441233325 2024-11-24T09:40:35,095 DEBUG [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,35917,1732441233385 2024-11-24T09:40:35,095 WARN [RS:1;ba35ada4783f:42723 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:40:35,095 INFO [RS:1;ba35ada4783f:42723 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:35,097 DEBUG [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325 2024-11-24T09:40:35,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T09:40:35,108 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:35,108 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba35ada4783f,42723,1732441233325] 2024-11-24T09:40:35,108 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba35ada4783f,35917,1732441233385] 2024-11-24T09:40:35,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:35,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T09:40:35,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T09:40:35,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:35,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:35,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T09:40:35,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T09:40:35,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:35,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:35,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T09:40:35,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740 2024-11-24T09:40:35,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740 2024-11-24T09:40:35,134 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T09:40:35,134 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T09:40:35,135 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:40:35,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T09:40:35,146 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T09:40:35,146 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T09:40:35,152 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba35ada4783f,43385,1732441231965 with port=36811, startcode=1732441233150 2024-11-24T09:40:35,154 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba35ada4783f,36811,1732441233150 2024-11-24T09:40:35,154 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] master.ServerManager(517): Registering regionserver=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:35,159 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63 2024-11-24T09:40:35,159 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43013 2024-11-24T09:40:35,159 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T09:40:35,169 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T09:40:35,169 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T09:40:35,171 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:35,172 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65762699, jitterRate=-0.02005942165851593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:40:35,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732441235086Initializing all the Stores at 1732441235088 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441235088Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441235088Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441235088Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441235088Cleaning up temporary data from old regions at 1732441235134 (+46 ms)Region opened successfully at 1732441235174 (+40 ms) 2024-11-24T09:40:35,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:40:35,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T09:40:35,175 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T09:40:35,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T09:40:35,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T09:40:35,175 DEBUG [RS:0;ba35ada4783f:36811 {}] zookeeper.ZKUtil(111): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba35ada4783f,36811,1732441233150 2024-11-24T09:40:35,175 INFO [RS:2;ba35ada4783f:35917 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:40:35,176 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba35ada4783f,36811,1732441233150] 2024-11-24T09:40:35,176 WARN [RS:0;ba35ada4783f:36811 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:40:35,176 INFO [RS:0;ba35ada4783f:36811 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:35,176 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,176 INFO [RS:1;ba35ada4783f:42723 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:40:35,176 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,176 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,36811,1732441233150 2024-11-24T09:40:35,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T09:40:35,183 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T09:40:35,184 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T09:40:35,188 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T09:40:35,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732441235174Disabling compacts and flushes for region at 1732441235174Disabling writes for close at 1732441235175 (+1 ms)Writing region close event to WAL at 1732441235188 (+13 ms)Closed at 1732441235188 2024-11-24T09:40:35,191 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T09:40:35,193 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,193 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T09:40:35,193 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,194 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,194 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,194 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:40:35,194 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,194 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T09:40:35,194 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,194 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,194 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,194 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,194 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,195 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:40:35,195 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,195 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,195 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,195 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,195 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,195 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,196 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:40:35,196 DEBUG [RS:2;ba35ada4783f:35917 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:40:35,196 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,197 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,197 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:40:35,197 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,197 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,197 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,197 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,197 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,197 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,198 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:40:35,198 DEBUG [RS:1;ba35ada4783f:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:40:35,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T09:40:35,214 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:40:35,220 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T09:40:35,223 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T09:40:35,227 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,227 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,227 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,227 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,228 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,228 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,35917,1732441233385-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:40:35,235 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,235 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,235 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,235 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,235 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,235 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,42723,1732441233325-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:40:35,243 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T09:40:35,255 INFO [RS:0;ba35ada4783f:36811 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:40:35,255 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,266 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T09:40:35,269 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T09:40:35,269 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,269 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,269 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,270 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,270 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,270 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,270 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T09:40:35,270 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:40:35,270 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,270 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,270 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,270 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,271 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,271 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:40:35,271 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:40:35,271 DEBUG [RS:0;ba35ada4783f:36811 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:40:35,272 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,35917,1732441233385-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,273 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,273 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.Replication(171): ba35ada4783f,35917,1732441233385 started 2024-11-24T09:40:35,296 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,296 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,296 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,296 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,296 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,296 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,36811,1732441233150-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:40:35,312 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T09:40:35,313 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,42723,1732441233325-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,313 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,314 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.Replication(171): ba35ada4783f,42723,1732441233325 started 2024-11-24T09:40:35,340 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,341 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(1482): Serving as ba35ada4783f,35917,1732441233385, RpcServer on ba35ada4783f/172.17.0.2:35917, sessionid=0x1016d80e1650003 2024-11-24T09:40:35,342 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T09:40:35,342 DEBUG [RS:2;ba35ada4783f:35917 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba35ada4783f,35917,1732441233385 2024-11-24T09:40:35,343 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,35917,1732441233385' 2024-11-24T09:40:35,343 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T09:40:35,344 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T09:40:35,345 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T09:40:35,345 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T09:40:35,345 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,36811,1732441233150-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,345 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T09:40:35,345 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,345 DEBUG [RS:2;ba35ada4783f:35917 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba35ada4783f,35917,1732441233385 2024-11-24T09:40:35,345 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,35917,1732441233385' 2024-11-24T09:40:35,345 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T09:40:35,346 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.Replication(171): ba35ada4783f,36811,1732441233150 started 2024-11-24T09:40:35,346 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T09:40:35,347 DEBUG [RS:2;ba35ada4783f:35917 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T09:40:35,347 INFO [RS:2;ba35ada4783f:35917 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T09:40:35,348 INFO [RS:2;ba35ada4783f:35917 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T09:40:35,354 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,354 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1482): Serving as ba35ada4783f,42723,1732441233325, RpcServer on ba35ada4783f/172.17.0.2:42723, sessionid=0x1016d80e1650002 2024-11-24T09:40:35,355 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T09:40:35,355 DEBUG [RS:1;ba35ada4783f:42723 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba35ada4783f,42723,1732441233325 2024-11-24T09:40:35,355 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,42723,1732441233325' 2024-11-24T09:40:35,355 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T09:40:35,356 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T09:40:35,356 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T09:40:35,356 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T09:40:35,356 DEBUG [RS:1;ba35ada4783f:42723 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba35ada4783f,42723,1732441233325 2024-11-24T09:40:35,356 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,42723,1732441233325' 2024-11-24T09:40:35,356 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T09:40:35,358 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T09:40:35,358 DEBUG [RS:1;ba35ada4783f:42723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T09:40:35,359 INFO [RS:1;ba35ada4783f:42723 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T09:40:35,359 INFO [RS:1;ba35ada4783f:42723 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T09:40:35,374 WARN [ba35ada4783f:43385 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T09:40:35,393 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:35,407 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1482): Serving as ba35ada4783f,36811,1732441233150, RpcServer on ba35ada4783f/172.17.0.2:36811, sessionid=0x1016d80e1650001 2024-11-24T09:40:35,407 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T09:40:35,407 DEBUG [RS:0;ba35ada4783f:36811 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba35ada4783f,36811,1732441233150 2024-11-24T09:40:35,407 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,36811,1732441233150' 2024-11-24T09:40:35,407 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T09:40:35,408 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T09:40:35,409 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T09:40:35,409 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T09:40:35,409 DEBUG [RS:0;ba35ada4783f:36811 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba35ada4783f,36811,1732441233150 2024-11-24T09:40:35,409 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,36811,1732441233150' 2024-11-24T09:40:35,409 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T09:40:35,410 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T09:40:35,411 DEBUG [RS:0;ba35ada4783f:36811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T09:40:35,412 INFO [RS:0;ba35ada4783f:36811 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T09:40:35,412 INFO [RS:0;ba35ada4783f:36811 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T09:40:35,455 INFO [RS:2;ba35ada4783f:35917 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:35,460 INFO [RS:1;ba35ada4783f:42723 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:35,460 INFO [RS:2;ba35ada4783f:35917 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C35917%2C1732441233385, suffix=, logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,35917,1732441233385, archiveDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs, maxLogs=32 2024-11-24T09:40:35,465 INFO [RS:1;ba35ada4783f:42723 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C42723%2C1732441233325, suffix=, logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325, archiveDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs, maxLogs=32 2024-11-24T09:40:35,492 DEBUG [RS:2;ba35ada4783f:35917 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,35917,1732441233385/ba35ada4783f%2C35917%2C1732441233385.1732441235464, exclude list is [], retry=0 2024-11-24T09:40:35,500 DEBUG [RS:1;ba35ada4783f:42723 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325/ba35ada4783f%2C42723%2C1732441233325.1732441235469, exclude list is [], retry=0 2024-11-24T09:40:35,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:35,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:35,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:35,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:35,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:35,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:35,513 INFO [RS:0;ba35ada4783f:36811 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:35,523 INFO [RS:0;ba35ada4783f:36811 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C36811%2C1732441233150, suffix=, logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,36811,1732441233150, archiveDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs, maxLogs=32 2024-11-24T09:40:35,549 DEBUG [RS:0;ba35ada4783f:36811 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,36811,1732441233150/ba35ada4783f%2C36811%2C1732441233150.1732441235525, exclude list is [], retry=0 2024-11-24T09:40:35,552 INFO [RS:1;ba35ada4783f:42723 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325/ba35ada4783f%2C42723%2C1732441233325.1732441235469 2024-11-24T09:40:35,555 INFO [RS:2;ba35ada4783f:35917 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,35917,1732441233385/ba35ada4783f%2C35917%2C1732441233385.1732441235464 2024-11-24T09:40:35,556 DEBUG [RS:1;ba35ada4783f:42723 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:40:35,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:35,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:35,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:35,563 DEBUG [RS:2;ba35ada4783f:35917 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:35,589 INFO [RS:0;ba35ada4783f:36811 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,36811,1732441233150/ba35ada4783f%2C36811%2C1732441233150.1732441235525 2024-11-24T09:40:35,590 DEBUG [RS:0;ba35ada4783f:36811 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:35,626 DEBUG [ba35ada4783f:43385 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T09:40:35,634 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(204): Hosts are {ba35ada4783f=0} racks are {/default-rack=0} 2024-11-24T09:40:35,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T09:40:35,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T09:40:35,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T09:40:35,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T09:40:35,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T09:40:35,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T09:40:35,642 INFO [ba35ada4783f:43385 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T09:40:35,642 INFO [ba35ada4783f:43385 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T09:40:35,642 INFO [ba35ada4783f:43385 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T09:40:35,642 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T09:40:35,650 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:35,655 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,42723,1732441233325, state=OPENING 2024-11-24T09:40:35,700 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T09:40:35,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:35,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:35,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:35,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:40:35,711 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:35,711 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:40:35,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,42723,1732441233325}] 2024-11-24T09:40:35,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:35,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:35,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:35,891 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:40:35,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50855, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:40:35,906 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T09:40:35,907 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:35,907 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T09:40:35,911 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C42723%2C1732441233325.meta, suffix=.meta, logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325, archiveDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs, maxLogs=32 2024-11-24T09:40:35,926 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, exclude list is [], retry=0 2024-11-24T09:40:35,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:35,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:35,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:35,936 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta 2024-11-24T09:40:35,937 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:35,937 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:35,939 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T09:40:35,942 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T09:40:35,949 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T09:40:35,955 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T09:40:35,956 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:35,956 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T09:40:35,956 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T09:40:35,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T09:40:35,962 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T09:40:35,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:35,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:35,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T09:40:35,966 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T09:40:35,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:35,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:35,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T09:40:35,969 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T09:40:35,970 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:35,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:35,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T09:40:35,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T09:40:35,973 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:35,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:35,975 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T09:40:35,976 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740 2024-11-24T09:40:35,980 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740 2024-11-24T09:40:35,983 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T09:40:35,984 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T09:40:35,985 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:40:35,989 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T09:40:35,991 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66862121, jitterRate=-0.0036767572164535522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:40:35,991 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T09:40:35,992 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732441235957Writing region info on filesystem at 1732441235957Initializing all the Stores at 1732441235960 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441235960Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441235960Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441235960Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441235960Cleaning up temporary data from old regions at 1732441235984 (+24 ms)Running coprocessor post-open hooks at 1732441235991 (+7 ms)Region opened successfully at 1732441235992 (+1 ms) 2024-11-24T09:40:35,998 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732441235882 2024-11-24T09:40:36,008 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T09:40:36,009 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T09:40:36,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:36,013 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,42723,1732441233325, state=OPEN 2024-11-24T09:40:36,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:36,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:36,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:36,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:36,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:36,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:36,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:36,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:36,042 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:36,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T09:40:36,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,42723,1732441233325 in 329 msec 2024-11-24T09:40:36,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T09:40:36,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 849 msec 2024-11-24T09:40:36,057 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:40:36,058 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T09:40:36,079 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:40:36,080 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1] 2024-11-24T09:40:36,102 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:40:36,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33117, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:40:36,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3600 sec 2024-11-24T09:40:36,129 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732441236129, completionTime=-1 2024-11-24T09:40:36,133 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T09:40:36,133 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T09:40:36,158 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T09:40:36,158 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732441296158 2024-11-24T09:40:36,159 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732441356159 2024-11-24T09:40:36,159 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-24T09:40:36,161 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T09:40:36,170 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,43385,1732441231965-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:36,171 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,43385,1732441231965-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:36,171 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,43385,1732441231965-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:36,172 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba35ada4783f:43385, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:36,173 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:36,173 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:36,180 DEBUG [master/ba35ada4783f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T09:40:36,206 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.610sec 2024-11-24T09:40:36,208 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T09:40:36,209 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T09:40:36,210 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T09:40:36,211 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T09:40:36,211 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T09:40:36,212 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,43385,1732441231965-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:40:36,213 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,43385,1732441231965-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T09:40:36,217 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T09:40:36,218 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T09:40:36,219 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,43385,1732441231965-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:36,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc48019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:40:36,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba35ada4783f,43385,-1 for getting cluster id 2024-11-24T09:40:36,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T09:40:36,339 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f30c0795-04ef-4a63-a29c-8fba537cfc7d' 2024-11-24T09:40:36,341 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T09:40:36,341 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f30c0795-04ef-4a63-a29c-8fba537cfc7d" 2024-11-24T09:40:36,342 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255ce710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:40:36,342 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba35ada4783f,43385,-1] 2024-11-24T09:40:36,344 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T09:40:36,345 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:40:36,347 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T09:40:36,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba7eb85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:40:36,350 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:40:36,357 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1] 2024-11-24T09:40:36,358 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:40:36,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:40:36,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba35ada4783f,43385,1732441231965 2024-11-24T09:40:36,398 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:43013/hbase 2024-11-24T09:40:36,416 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=8085 2024-11-24T09:40:36,439 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:36,442 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:36,443 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:36,447 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-08785746, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-08785746, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:36,460 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-08785746/hregion-08785746.1732441236448, exclude list is [], retry=0 2024-11-24T09:40:36,464 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:36,464 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:36,465 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:36,468 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-08785746/hregion-08785746.1732441236448 2024-11-24T09:40:36,468 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:40:36,468 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e9e0cb74d5be89918ad40eeb2df9dc80, NAME => 'testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:40:36,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741838_1014 (size=64) 2024-11-24T09:40:36,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741838_1014 (size=64) 2024-11-24T09:40:36,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741838_1014 (size=64) 2024-11-24T09:40:36,483 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:36,485 INFO [StoreOpener-e9e0cb74d5be89918ad40eeb2df9dc80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,487 INFO [StoreOpener-e9e0cb74d5be89918ad40eeb2df9dc80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9e0cb74d5be89918ad40eeb2df9dc80 columnFamilyName a 2024-11-24T09:40:36,487 DEBUG [StoreOpener-e9e0cb74d5be89918ad40eeb2df9dc80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:36,488 INFO [StoreOpener-e9e0cb74d5be89918ad40eeb2df9dc80-1 {}] regionserver.HStore(327): Store=e9e0cb74d5be89918ad40eeb2df9dc80/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:36,488 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,489 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,490 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,491 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,491 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,493 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,496 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:36,497 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9e0cb74d5be89918ad40eeb2df9dc80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69350841, jitterRate=0.03340806066989899}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:40:36,498 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9e0cb74d5be89918ad40eeb2df9dc80: Writing region info on filesystem at 1732441236484Initializing all the Stores at 1732441236485 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441236485Cleaning up temporary data from old regions at 1732441236491 (+6 ms)Region opened successfully at 1732441236498 (+7 ms) 2024-11-24T09:40:36,498 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e9e0cb74d5be89918ad40eeb2df9dc80, disabling compactions & flushes 2024-11-24T09:40:36,498 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80. 2024-11-24T09:40:36,498 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80. 2024-11-24T09:40:36,498 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80. after waiting 0 ms 2024-11-24T09:40:36,498 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80. 2024-11-24T09:40:36,499 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80. 2024-11-24T09:40:36,499 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e9e0cb74d5be89918ad40eeb2df9dc80: Waiting for close lock at 1732441236498Disabling compacts and flushes for region at 1732441236498Disabling writes for close at 1732441236498Writing region close event to WAL at 1732441236499 (+1 ms)Closed at 1732441236499 2024-11-24T09:40:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741837_1013 (size=93) 2024-11-24T09:40:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741837_1013 (size=93) 2024-11-24T09:40:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741837_1013 (size=93) 2024-11-24T09:40:36,509 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:36,509 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-08785746:(num 1732441236448) 2024-11-24T09:40:36,511 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T09:40:36,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741839_1015 (size=276) 2024-11-24T09:40:36,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741839_1015 (size=276) 2024-11-24T09:40:36,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741839_1015 (size=276) 2024-11-24T09:40:36,529 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T09:40:36,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741840_1016 (size=230) 2024-11-24T09:40:36,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741840_1016 (size=230) 2024-11-24T09:40:36,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741840_1016 (size=230) 2024-11-24T09:40:36,561 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1, size=276 (276bytes) 2024-11-24T09:40:36,562 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T09:40:36,562 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T09:40:36,562 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1 2024-11-24T09:40:36,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1 after 3ms 2024-11-24T09:40:36,571 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:36,572 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1 took 12ms 2024-11-24T09:40:36,576 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1 so closing down 2024-11-24T09:40:36,579 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T09:40:36,580 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:36,581 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T09:40:36,583 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:36,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741841_1017 (size=276) 2024-11-24T09:40:36,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741841_1017 (size=276) 2024-11-24T09:40:36,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741841_1017 (size=276) 2024-11-24T09:40:36,595 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T09:40:36,598 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002 2024-11-24T09:40:36,603 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-24T09:40:36,604 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1, journal: Splitting hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1, size=276 (276bytes) at 1732441236561Finishing writing output for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1 so closing down at 1732441236576 (+15 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000001-wal-1.temp at 1732441236581 (+5 ms)3 split writer threads finished at 1732441236583 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732441236595 (+12 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002 at 1732441236598 (+3 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732441236603 (+5 ms) 2024-11-24T09:40:36,622 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2, size=230 (230bytes) 2024-11-24T09:40:36,622 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2 2024-11-24T09:40:36,622 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2 after 0ms 2024-11-24T09:40:36,627 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:36,627 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2 took 6ms 2024-11-24T09:40:36,631 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2 so closing down 2024-11-24T09:40:36,631 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:36,634 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T09:40:36,636 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T09:40:36,637 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:36,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741842_1018 (size=230) 2024-11-24T09:40:36,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741842_1018 (size=230) 2024-11-24T09:40:36,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741842_1018 (size=230) 2024-11-24T09:40:36,647 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T09:40:36,651 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:36,654 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-11-24T09:40:36,655 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-24T09:40:36,655 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2, journal: Splitting hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2, size=230 (230bytes) at 1732441236622Finishing writing output for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2 so closing down at 1732441236631 (+9 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002-wal-2.temp at 1732441236636 (+5 ms)3 split writer threads finished at 1732441236637 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732441236647 (+10 ms)Processed 1 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732441236655 (+8 ms) 2024-11-24T09:40:36,656 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:36,658 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:36,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal.1732441236658, exclude list is [], retry=0 2024-11-24T09:40:36,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:36,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:36,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:36,678 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal.1732441236658 2024-11-24T09:40:36,678 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:36,679 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e9e0cb74d5be89918ad40eeb2df9dc80, NAME => 'testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:36,679 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:36,679 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,679 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,681 INFO [StoreOpener-e9e0cb74d5be89918ad40eeb2df9dc80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,683 INFO [StoreOpener-e9e0cb74d5be89918ad40eeb2df9dc80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9e0cb74d5be89918ad40eeb2df9dc80 columnFamilyName a 2024-11-24T09:40:36,683 DEBUG [StoreOpener-e9e0cb74d5be89918ad40eeb2df9dc80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:36,685 INFO [StoreOpener-e9e0cb74d5be89918ad40eeb2df9dc80-1 {}] regionserver.HStore(327): Store=e9e0cb74d5be89918ad40eeb2df9dc80/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:36,685 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,686 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,690 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,691 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002 2024-11-24T09:40:36,696 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:36,702 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002 2024-11-24T09:40:36,705 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e9e0cb74d5be89918ad40eeb2df9dc80 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T09:40:36,762 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/.tmp/a/257a4b48fea344f9b41efea04a1fa52f is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732441236510/Put/seqid=0 2024-11-24T09:40:36,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741844_1020 (size=5170) 2024-11-24T09:40:36,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741844_1020 (size=5170) 2024-11-24T09:40:36,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741844_1020 (size=5170) 2024-11-24T09:40:36,777 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/.tmp/a/257a4b48fea344f9b41efea04a1fa52f 2024-11-24T09:40:36,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741828_1004 (size=1189) 2024-11-24T09:40:36,828 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/.tmp/a/257a4b48fea344f9b41efea04a1fa52f as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/a/257a4b48fea344f9b41efea04a1fa52f 2024-11-24T09:40:36,837 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/a/257a4b48fea344f9b41efea04a1fa52f, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T09:40:36,842 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e9e0cb74d5be89918ad40eeb2df9dc80 in 135ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T09:40:36,844 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/0000000000000000002 2024-11-24T09:40:36,845 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,845 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,849 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9e0cb74d5be89918ad40eeb2df9dc80 2024-11-24T09:40:36,852 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/e9e0cb74d5be89918ad40eeb2df9dc80/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T09:40:36,853 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9e0cb74d5be89918ad40eeb2df9dc80; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67062140, jitterRate=-6.962418556213379E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:40:36,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9e0cb74d5be89918ad40eeb2df9dc80: Writing region info on filesystem at 1732441236679Initializing all the Stores at 1732441236681 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441236681Obtaining lock to block concurrent updates at 1732441236705 (+24 ms)Preparing flush snapshotting stores in e9e0cb74d5be89918ad40eeb2df9dc80 at 1732441236705Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732441236708 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732441236440.e9e0cb74d5be89918ad40eeb2df9dc80. at 1732441236708Flushing e9e0cb74d5be89918ad40eeb2df9dc80/a: creating writer at 1732441236709 (+1 ms)Flushing e9e0cb74d5be89918ad40eeb2df9dc80/a: appending metadata at 1732441236750 (+41 ms)Flushing e9e0cb74d5be89918ad40eeb2df9dc80/a: closing flushed file at 1732441236753 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c0a0d29: reopening flushed file at 1732441236826 (+73 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e9e0cb74d5be89918ad40eeb2df9dc80 in 135ms, sequenceid=2, compaction requested=false; wal=null at 1732441236843 (+17 ms)Cleaning up temporary data from old regions at 1732441236845 (+2 ms)Region opened successfully at 1732441236855 (+10 ms) 2024-11-24T09:40:36,876 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=374 (was 362) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:57650 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:55930 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:57546 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@6bc52d8f[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:53588 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:56084 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:53430 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:43013/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=8072 (was 8085) 2024-11-24T09:40:36,886 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=8072 2024-11-24T09:40:36,905 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:36,907 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:36,908 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:36,912 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-05411332, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-05411332, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:36,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741827_1003 (size=196) 2024-11-24T09:40:36,925 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-05411332/hregion-05411332.1732441236913, exclude list is [], retry=0 2024-11-24T09:40:36,929 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:36,929 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:36,929 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:36,932 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-05411332/hregion-05411332.1732441236913 2024-11-24T09:40:36,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:40:36,933 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 511a5cca9a4d307c672aefffe5b22da6, NAME => 'testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:40:36,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741846_1022 (size=64) 2024-11-24T09:40:36,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741846_1022 (size=64) 2024-11-24T09:40:36,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741846_1022 (size=64) 2024-11-24T09:40:36,945 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:36,947 INFO [StoreOpener-511a5cca9a4d307c672aefffe5b22da6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:36,950 INFO [StoreOpener-511a5cca9a4d307c672aefffe5b22da6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 511a5cca9a4d307c672aefffe5b22da6 columnFamilyName a 2024-11-24T09:40:36,950 DEBUG [StoreOpener-511a5cca9a4d307c672aefffe5b22da6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:36,951 INFO [StoreOpener-511a5cca9a4d307c672aefffe5b22da6-1 {}] regionserver.HStore(327): Store=511a5cca9a4d307c672aefffe5b22da6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:36,951 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:36,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:36,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:36,954 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:36,954 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:36,957 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:36,960 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:36,961 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 511a5cca9a4d307c672aefffe5b22da6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70488627, jitterRate=0.050362393260002136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:40:36,961 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 511a5cca9a4d307c672aefffe5b22da6: Writing region info on filesystem at 1732441236945Initializing all the Stores at 1732441236947 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441236947Cleaning up temporary data from old regions at 1732441236954 (+7 ms)Region opened successfully at 1732441236961 (+7 ms) 2024-11-24T09:40:36,961 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 511a5cca9a4d307c672aefffe5b22da6, disabling compactions & flushes 2024-11-24T09:40:36,961 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6. 2024-11-24T09:40:36,962 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6. 2024-11-24T09:40:36,962 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6. after waiting 0 ms 2024-11-24T09:40:36,962 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6. 2024-11-24T09:40:36,962 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6. 2024-11-24T09:40:36,962 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 511a5cca9a4d307c672aefffe5b22da6: Waiting for close lock at 1732441236961Disabling compacts and flushes for region at 1732441236961Disabling writes for close at 1732441236962 (+1 ms)Writing region close event to WAL at 1732441236962Closed at 1732441236962 2024-11-24T09:40:36,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741845_1021 (size=93) 2024-11-24T09:40:36,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741845_1021 (size=93) 2024-11-24T09:40:36,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741845_1021 (size=93) 2024-11-24T09:40:36,969 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:36,969 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-05411332:(num 1732441236913) 2024-11-24T09:40:36,970 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T09:40:36,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741847_1023 (size=276) 2024-11-24T09:40:36,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741847_1023 (size=276) 2024-11-24T09:40:36,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741847_1023 (size=276) 2024-11-24T09:40:36,982 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T09:40:36,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741848_1024 (size=230) 2024-11-24T09:40:36,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741848_1024 (size=230) 2024-11-24T09:40:36,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741848_1024 (size=230) 2024-11-24T09:40:37,008 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2, size=230 (230bytes) 2024-11-24T09:40:37,008 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2 2024-11-24T09:40:37,009 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2 after 1ms 2024-11-24T09:40:37,013 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:37,013 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2 took 6ms 2024-11-24T09:40:37,016 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2 so closing down 2024-11-24T09:40:37,016 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:37,018 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T09:40:37,020 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T09:40:37,020 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:37,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741849_1025 (size=230) 2024-11-24T09:40:37,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741849_1025 (size=230) 2024-11-24T09:40:37,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741849_1025 (size=230) 2024-11-24T09:40:37,040 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T09:40:37,042 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002 2024-11-24T09:40:37,042 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-24T09:40:37,042 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2, journal: Splitting hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2, size=230 (230bytes) at 1732441237008Finishing writing output for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2 so closing down at 1732441237016 (+8 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002-wal-2.temp at 1732441237020 (+4 ms)3 split writer threads finished at 1732441237021 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732441237040 (+19 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002 at 1732441237042 (+2 ms)Processed 1 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732441237042 2024-11-24T09:40:37,059 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1, size=276 (276bytes) 2024-11-24T09:40:37,059 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1 2024-11-24T09:40:37,060 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1 after 0ms 2024-11-24T09:40:37,065 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:37,065 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1 took 6ms 2024-11-24T09:40:37,068 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1 so closing down 2024-11-24T09:40:37,068 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:37,071 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T09:40:37,073 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T09:40:37,073 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:37,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741850_1026 (size=276) 2024-11-24T09:40:37,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741850_1026 (size=276) 2024-11-24T09:40:37,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741850_1026 (size=276) 2024-11-24T09:40:37,087 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T09:40:37,091 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:37,094 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002, length=230 2024-11-24T09:40:37,096 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002 2024-11-24T09:40:37,096 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-24T09:40:37,096 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1, journal: Splitting hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1, size=276 (276bytes) at 1732441237059Finishing writing output for hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1 so closing down at 1732441237068 (+9 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000001-wal-1.temp at 1732441237073 (+5 ms)3 split writer threads finished at 1732441237073Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732441237087 (+14 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002 at 1732441237096 (+9 ms)Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732441237096 2024-11-24T09:40:37,096 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:37,098 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:37,110 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal.1732441237099, exclude list is [], retry=0 2024-11-24T09:40:37,114 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:37,114 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:37,115 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:37,117 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal.1732441237099 2024-11-24T09:40:37,117 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:37,117 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 511a5cca9a4d307c672aefffe5b22da6, NAME => 'testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:37,117 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:37,117 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,117 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,119 INFO [StoreOpener-511a5cca9a4d307c672aefffe5b22da6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,121 INFO [StoreOpener-511a5cca9a4d307c672aefffe5b22da6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 511a5cca9a4d307c672aefffe5b22da6 columnFamilyName a 2024-11-24T09:40:37,121 DEBUG [StoreOpener-511a5cca9a4d307c672aefffe5b22da6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:37,122 INFO [StoreOpener-511a5cca9a4d307c672aefffe5b22da6-1 {}] regionserver.HStore(327): Store=511a5cca9a4d307c672aefffe5b22da6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:37,122 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,123 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,125 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,126 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002 2024-11-24T09:40:37,129 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:37,130 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002 2024-11-24T09:40:37,131 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 511a5cca9a4d307c672aefffe5b22da6 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T09:40:37,147 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/.tmp/a/daa7f52a4281487d8b6722cade89b0b8 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732441236969/Put/seqid=0 2024-11-24T09:40:37,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741852_1028 (size=5170) 2024-11-24T09:40:37,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741852_1028 (size=5170) 2024-11-24T09:40:37,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741852_1028 (size=5170) 2024-11-24T09:40:37,160 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/.tmp/a/daa7f52a4281487d8b6722cade89b0b8 2024-11-24T09:40:37,170 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/.tmp/a/daa7f52a4281487d8b6722cade89b0b8 as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/a/daa7f52a4281487d8b6722cade89b0b8 2024-11-24T09:40:37,179 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/a/daa7f52a4281487d8b6722cade89b0b8, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T09:40:37,179 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 511a5cca9a4d307c672aefffe5b22da6 in 48ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T09:40:37,180 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/0000000000000000002 2024-11-24T09:40:37,181 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,181 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,183 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 511a5cca9a4d307c672aefffe5b22da6 2024-11-24T09:40:37,186 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/511a5cca9a4d307c672aefffe5b22da6/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T09:40:37,187 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 511a5cca9a4d307c672aefffe5b22da6; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69592600, jitterRate=0.0370105504989624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:40:37,188 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 511a5cca9a4d307c672aefffe5b22da6: Writing region info on filesystem at 1732441237118Initializing all the Stores at 1732441237119 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441237119Obtaining lock to block concurrent updates at 1732441237131 (+12 ms)Preparing flush snapshotting stores in 511a5cca9a4d307c672aefffe5b22da6 at 1732441237131Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732441237131Flushing stores of testReplayEditsWrittenIntoWAL,,1732441236905.511a5cca9a4d307c672aefffe5b22da6. at 1732441237131Flushing 511a5cca9a4d307c672aefffe5b22da6/a: creating writer at 1732441237131Flushing 511a5cca9a4d307c672aefffe5b22da6/a: appending metadata at 1732441237146 (+15 ms)Flushing 511a5cca9a4d307c672aefffe5b22da6/a: closing flushed file at 1732441237146Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e5781df: reopening flushed file at 1732441237169 (+23 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 511a5cca9a4d307c672aefffe5b22da6 in 48ms, sequenceid=2, compaction requested=false; wal=null at 1732441237179 (+10 ms)Cleaning up temporary data from old regions at 1732441237181 (+2 ms)Region opened successfully at 1732441237187 (+6 ms) 2024-11-24T09:40:37,202 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=385 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:55930 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:56152 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:53430 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:57708 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:57546 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@294068bf[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:53678 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=8065 (was 8072) 2024-11-24T09:40:37,214 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=8064 2024-11-24T09:40:37,231 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:37,232 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:37,233 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:37,236 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63248876, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-63248876, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:37,250 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63248876/hregion-63248876.1732441237237, exclude list is [], retry=0 2024-11-24T09:40:37,253 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:37,253 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:37,254 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:37,256 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63248876/hregion-63248876.1732441237237 2024-11-24T09:40:37,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:37,257 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ecaae86e296bd4b5f1b2919ecae8cd9e, NAME => 'testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:40:37,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741854_1030 (size=64) 2024-11-24T09:40:37,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741854_1030 (size=64) 2024-11-24T09:40:37,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741854_1030 (size=64) 2024-11-24T09:40:37,271 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:37,273 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,274 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecaae86e296bd4b5f1b2919ecae8cd9e columnFamilyName a 2024-11-24T09:40:37,274 DEBUG [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:37,275 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(327): Store=ecaae86e296bd4b5f1b2919ecae8cd9e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:37,275 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,277 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecaae86e296bd4b5f1b2919ecae8cd9e columnFamilyName b 2024-11-24T09:40:37,277 DEBUG [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:37,277 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(327): Store=ecaae86e296bd4b5f1b2919ecae8cd9e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:37,277 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,279 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecaae86e296bd4b5f1b2919ecae8cd9e columnFamilyName c 2024-11-24T09:40:37,279 DEBUG [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:37,280 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(327): Store=ecaae86e296bd4b5f1b2919ecae8cd9e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:37,280 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,281 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,281 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,283 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,283 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,284 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:37,285 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:37,288 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:37,288 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ecaae86e296bd4b5f1b2919ecae8cd9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69418629, jitterRate=0.03441818058490753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:37,289 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ecaae86e296bd4b5f1b2919ecae8cd9e: Writing region info on filesystem at 1732441237271Initializing all the Stores at 1732441237272 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441237272Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441237272Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441237272Cleaning up temporary data from old regions at 1732441237283 (+11 ms)Region opened successfully at 1732441237288 (+5 ms) 2024-11-24T09:40:37,289 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ecaae86e296bd4b5f1b2919ecae8cd9e, disabling compactions & flushes 2024-11-24T09:40:37,289 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:37,289 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:37,289 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. after waiting 0 ms 2024-11-24T09:40:37,289 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:37,289 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:37,289 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ecaae86e296bd4b5f1b2919ecae8cd9e: Waiting for close lock at 1732441237289Disabling compacts and flushes for region at 1732441237289Disabling writes for close at 1732441237289Writing region close event to WAL at 1732441237289Closed at 1732441237289 2024-11-24T09:40:37,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741853_1029 (size=93) 2024-11-24T09:40:37,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741853_1029 (size=93) 2024-11-24T09:40:37,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741853_1029 (size=93) 2024-11-24T09:40:37,295 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:37,295 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63248876:(num 1732441237237) 2024-11-24T09:40:37,295 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:37,297 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:37,310 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297, exclude list is [], retry=0 2024-11-24T09:40:37,313 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:37,313 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:37,313 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:37,316 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 2024-11-24T09:40:37,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:37,536 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297, size=0 (0bytes) 2024-11-24T09:40:37,536 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 might be still open, length is 0 2024-11-24T09:40:37,536 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 2024-11-24T09:40:37,537 WARN [IPC Server handler 0 on default port 43013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T09:40:37,538 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 after 2ms 2024-11-24T09:40:39,930 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:56202 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45595:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56202 dst: /127.0.0.1:45595 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45595 remote=/127.0.0.1:56202]. Total timeout mills is 60000, 57569 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:39,931 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:53706 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43599:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53706 dst: /127.0.0.1:43599 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:39,931 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:57760 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45495:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57760 dst: /127.0.0.1:45495 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:39,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741855_1032 (size=200589) 2024-11-24T09:40:39,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741855_1032 (size=200589) 2024-11-24T09:40:39,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741855_1032 (size=200589) 2024-11-24T09:40:41,473 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T09:40:41,539 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 after 4003ms 2024-11-24T09:40:41,544 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:41,544 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T09:40:41,546 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 took 4010ms 2024-11-24T09:40:41,554 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732441237297.temp 2024-11-24T09:40:41,563 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000000001-wal.1732441237297.temp 2024-11-24T09:40:41,660 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297; continuing. 2024-11-24T09:40:41,660 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 so closing down 2024-11-24T09:40:41,660 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:41,662 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:41,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741856_1033 (size=200597) 2024-11-24T09:40:41,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741856_1033 (size=200597) 2024-11-24T09:40:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741856_1033 (size=200597) 2024-11-24T09:40:41,672 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000000001-wal.1732441237297.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-11-24T09:40:41,674 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000000001-wal.1732441237297.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000003002 2024-11-24T09:40:41,674 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 125 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T09:40:41,674 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297, journal: Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297, size=0 (0bytes) at 1732441237536Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000000001-wal.1732441237297.temp at 1732441241563 (+4027 ms)Split 1024 edits, skipped 0 edits. at 1732441241611 (+48 ms)Split 2048 edits, skipped 0 edits. at 1732441241636 (+25 ms)Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 so closing down at 1732441241660 (+24 ms)3 split writer threads finished at 1732441241662 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000000001-wal.1732441237297.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1732441241672 (+10 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000000001-wal.1732441237297.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000003002 at 1732441241674 (+2 ms)Processed 3002 edits across 1 Regions in 125 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297, size=0, length=0, corrupted=false, cancelled=false at 1732441241674 2024-11-24T09:40:41,677 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441237297 2024-11-24T09:40:41,679 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000003002 2024-11-24T09:40:41,679 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:41,681 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:41,695 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441241682, exclude list is [], retry=0 2024-11-24T09:40:41,699 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:41,699 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:41,700 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:41,702 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441241682 2024-11-24T09:40:41,703 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:40:41,703 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:41,708 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:41,710 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecaae86e296bd4b5f1b2919ecae8cd9e columnFamilyName a 2024-11-24T09:40:41,710 DEBUG [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:41,711 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(327): Store=ecaae86e296bd4b5f1b2919ecae8cd9e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:41,711 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:41,712 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecaae86e296bd4b5f1b2919ecae8cd9e columnFamilyName b 2024-11-24T09:40:41,712 DEBUG [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:41,713 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(327): Store=ecaae86e296bd4b5f1b2919ecae8cd9e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:41,713 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:41,714 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecaae86e296bd4b5f1b2919ecae8cd9e columnFamilyName c 2024-11-24T09:40:41,714 DEBUG [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:41,715 INFO [StoreOpener-ecaae86e296bd4b5f1b2919ecae8cd9e-1 {}] regionserver.HStore(327): Store=ecaae86e296bd4b5f1b2919ecae8cd9e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:41,715 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:41,716 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:41,719 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:41,719 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000003002 2024-11-24T09:40:41,723 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:41,761 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T09:40:42,116 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ecaae86e296bd4b5f1b2919ecae8cd9e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T09:40:42,161 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/a/52da02cbbc624c21a10ff8115644ec79 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732441237326/Put/seqid=0 2024-11-24T09:40:42,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741858_1035 (size=50463) 2024-11-24T09:40:42,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741858_1035 (size=50463) 2024-11-24T09:40:42,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741858_1035 (size=50463) 2024-11-24T09:40:42,194 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/a/52da02cbbc624c21a10ff8115644ec79 2024-11-24T09:40:42,206 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/a/52da02cbbc624c21a10ff8115644ec79 as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/a/52da02cbbc624c21a10ff8115644ec79 2024-11-24T09:40:42,216 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/a/52da02cbbc624c21a10ff8115644ec79, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T09:40:42,217 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for ecaae86e296bd4b5f1b2919ecae8cd9e in 101ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T09:40:42,252 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T09:40:42,253 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ecaae86e296bd4b5f1b2919ecae8cd9e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T09:40:42,311 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/a/36abfb9df4934c3b99049f1d275d7eb9 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732441237384/Put/seqid=0 2024-11-24T09:40:42,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741859_1036 (size=20072) 2024-11-24T09:40:42,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741859_1036 (size=20072) 2024-11-24T09:40:42,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741859_1036 (size=20072) 2024-11-24T09:40:42,390 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/a/36abfb9df4934c3b99049f1d275d7eb9 2024-11-24T09:40:42,427 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T09:40:42,428 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T09:40:42,429 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T09:40:42,430 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T09:40:42,430 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T09:40:42,430 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T09:40:42,431 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T09:40:42,431 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T09:40:42,461 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/b/cd55c5b925ca4219ac3aceb7cccb2dfd is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732441237406/Put/seqid=0 2024-11-24T09:40:42,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741860_1037 (size=35835) 2024-11-24T09:40:42,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741860_1037 (size=35835) 2024-11-24T09:40:42,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741860_1037 (size=35835) 2024-11-24T09:40:42,517 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/b/cd55c5b925ca4219ac3aceb7cccb2dfd 2024-11-24T09:40:42,549 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/a/36abfb9df4934c3b99049f1d275d7eb9 as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/a/36abfb9df4934c3b99049f1d275d7eb9 2024-11-24T09:40:42,560 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/a/36abfb9df4934c3b99049f1d275d7eb9, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T09:40:42,564 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/b/cd55c5b925ca4219ac3aceb7cccb2dfd as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/b/cd55c5b925ca4219ac3aceb7cccb2dfd 2024-11-24T09:40:42,585 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/b/cd55c5b925ca4219ac3aceb7cccb2dfd, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T09:40:42,587 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ecaae86e296bd4b5f1b2919ecae8cd9e in 332ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T09:40:42,630 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T09:40:42,630 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ecaae86e296bd4b5f1b2919ecae8cd9e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T09:40:42,666 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/b/9a0ec486ddef4d75a8886bbedc5c3107 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732441237424/Put/seqid=0 2024-11-24T09:40:42,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741861_1038 (size=35082) 2024-11-24T09:40:42,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741861_1038 (size=35082) 2024-11-24T09:40:42,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741861_1038 (size=35082) 2024-11-24T09:40:42,708 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/b/9a0ec486ddef4d75a8886bbedc5c3107 2024-11-24T09:40:42,762 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/c/42996f5fe39b406fb6b4043ee15b464c is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732441237452/Put/seqid=0 2024-11-24T09:40:42,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741862_1039 (size=20825) 2024-11-24T09:40:42,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741862_1039 (size=20825) 2024-11-24T09:40:42,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741862_1039 (size=20825) 2024-11-24T09:40:42,823 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/c/42996f5fe39b406fb6b4043ee15b464c 2024-11-24T09:40:42,847 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/b/9a0ec486ddef4d75a8886bbedc5c3107 as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/b/9a0ec486ddef4d75a8886bbedc5c3107 2024-11-24T09:40:42,858 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/b/9a0ec486ddef4d75a8886bbedc5c3107, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T09:40:42,860 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/c/42996f5fe39b406fb6b4043ee15b464c as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/c/42996f5fe39b406fb6b4043ee15b464c 2024-11-24T09:40:42,879 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/c/42996f5fe39b406fb6b4043ee15b464c, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T09:40:42,880 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ecaae86e296bd4b5f1b2919ecae8cd9e in 249ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T09:40:42,904 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732441237497/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:42,916 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000003002 2024-11-24T09:40:42,916 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T09:40:42,917 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ecaae86e296bd4b5f1b2919ecae8cd9e 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T09:40:42,964 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/c/81c62bb5912744adaaab7c2d9fca8aef is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732441237461/Put/seqid=0 2024-11-24T09:40:43,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741863_1040 (size=50301) 2024-11-24T09:40:43,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741863_1040 (size=50301) 2024-11-24T09:40:43,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741863_1040 (size=50301) 2024-11-24T09:40:43,054 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/c/81c62bb5912744adaaab7c2d9fca8aef 2024-11-24T09:40:43,065 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 81c62bb5912744adaaab7c2d9fca8aef 2024-11-24T09:40:43,067 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/.tmp/c/81c62bb5912744adaaab7c2d9fca8aef as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/c/81c62bb5912744adaaab7c2d9fca8aef 2024-11-24T09:40:43,078 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 81c62bb5912744adaaab7c2d9fca8aef 2024-11-24T09:40:43,078 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/c/81c62bb5912744adaaab7c2d9fca8aef, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T09:40:43,078 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for ecaae86e296bd4b5f1b2919ecae8cd9e in 162ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T09:40:43,080 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/0000000000000003002 2024-11-24T09:40:43,082 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:43,082 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:43,083 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T09:40:43,086 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ecaae86e296bd4b5f1b2919ecae8cd9e 2024-11-24T09:40:43,100 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenIntoWAL/ecaae86e296bd4b5f1b2919ecae8cd9e/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T09:40:43,102 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ecaae86e296bd4b5f1b2919ecae8cd9e; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72603018, jitterRate=0.08186927437782288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T09:40:43,102 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ecaae86e296bd4b5f1b2919ecae8cd9e: Writing region info on filesystem at 1732441241703Initializing all the Stores at 1732441241705 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441241705Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441241708 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441241708Cleaning up temporary data from old regions at 1732441243082 (+1374 ms)Region opened successfully at 1732441243102 (+20 ms) 2024-11-24T09:40:43,222 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ecaae86e296bd4b5f1b2919ecae8cd9e, disabling compactions & flushes 2024-11-24T09:40:43,222 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:43,223 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:43,223 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. after waiting 0 ms 2024-11-24T09:40:43,223 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:43,266 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732441237231.ecaae86e296bd4b5f1b2919ecae8cd9e. 2024-11-24T09:40:43,267 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ecaae86e296bd4b5f1b2919ecae8cd9e: Waiting for close lock at 1732441243222Disabling compacts and flushes for region at 1732441243222Disabling writes for close at 1732441243223 (+1 ms)Writing region close event to WAL at 1732441243266 (+43 ms)Closed at 1732441243266 2024-11-24T09:40:43,286 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441241682 not finished, retry = 0 2024-11-24T09:40:43,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741857_1034 (size=93) 2024-11-24T09:40:43,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741857_1034 (size=93) 2024-11-24T09:40:43,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741857_1034 (size=93) 2024-11-24T09:40:43,392 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:43,392 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441241682) 2024-11-24T09:40:43,425 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=401 (was 385) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1560450331_22 at /127.0.0.1:46640 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41433 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:43013 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (373550922) connection to localhost/127.0.0.1:43013 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36399 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (373550922) connection to localhost/127.0.0.1:36399 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (373550922) connection to localhost/127.0.0.1:41433 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1560450331_22 at /127.0.0.1:58474 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1560450331_22 at /127.0.0.1:53762 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1560450331_22 at /127.0.0.1:56220 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=571 (was 562) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7985 (was 8064) 2024-11-24T09:40:43,443 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=571, ProcessCount=11, AvailableMemoryMB=7984 2024-11-24T09:40:43,467 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:43,471 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:43,473 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:43,483 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-17885625, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-17885625, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:43,504 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-17885625/hregion-17885625.1732441243484, exclude list is [], retry=0 2024-11-24T09:40:43,525 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:43,530 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:43,551 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:43,591 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-17885625/hregion-17885625.1732441243484 2024-11-24T09:40:43,597 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:43,597 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b512189a5cb637b9f79cf1f7b8015a0e, NAME => 'test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:40:43,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741865_1042 (size=43) 2024-11-24T09:40:43,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741865_1042 (size=43) 2024-11-24T09:40:43,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741865_1042 (size=43) 2024-11-24T09:40:43,657 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:43,668 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,677 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b512189a5cb637b9f79cf1f7b8015a0e columnFamilyName a 2024-11-24T09:40:43,677 DEBUG [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:43,678 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(327): Store=b512189a5cb637b9f79cf1f7b8015a0e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:43,692 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,697 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b512189a5cb637b9f79cf1f7b8015a0e columnFamilyName b 2024-11-24T09:40:43,697 DEBUG [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:43,699 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(327): Store=b512189a5cb637b9f79cf1f7b8015a0e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:43,699 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,702 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b512189a5cb637b9f79cf1f7b8015a0e columnFamilyName c 2024-11-24T09:40:43,702 DEBUG [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:43,703 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(327): Store=b512189a5cb637b9f79cf1f7b8015a0e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:43,703 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,706 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,707 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,712 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,712 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,713 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:43,718 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:43,743 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:43,745 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b512189a5cb637b9f79cf1f7b8015a0e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71735967, jitterRate=0.06894920766353607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:43,746 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b512189a5cb637b9f79cf1f7b8015a0e: Writing region info on filesystem at 1732441243657Initializing all the Stores at 1732441243659 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441243659Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441243668 (+9 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441243668Cleaning up temporary data from old regions at 1732441243712 (+44 ms)Region opened successfully at 1732441243746 (+34 ms) 2024-11-24T09:40:43,746 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b512189a5cb637b9f79cf1f7b8015a0e, disabling compactions & flushes 2024-11-24T09:40:43,746 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. 2024-11-24T09:40:43,746 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. 2024-11-24T09:40:43,746 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. after waiting 0 ms 2024-11-24T09:40:43,746 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. 2024-11-24T09:40:43,747 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. 2024-11-24T09:40:43,747 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b512189a5cb637b9f79cf1f7b8015a0e: Waiting for close lock at 1732441243746Disabling compacts and flushes for region at 1732441243746Disabling writes for close at 1732441243746Writing region close event to WAL at 1732441243747 (+1 ms)Closed at 1732441243747 2024-11-24T09:40:43,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741864_1041 (size=93) 2024-11-24T09:40:43,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741864_1041 (size=93) 2024-11-24T09:40:43,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741864_1041 (size=93) 2024-11-24T09:40:43,778 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:43,778 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-17885625:(num 1732441243484) 2024-11-24T09:40:43,779 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:43,781 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:43,802 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782, exclude list is [], retry=0 2024-11-24T09:40:43,807 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:43,808 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:43,808 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:43,868 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782 2024-11-24T09:40:43,871 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:40:44,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741866_1043 (size=200357) 2024-11-24T09:40:44,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741866_1043 (size=200357) 2024-11-24T09:40:44,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741866_1043 (size=200357) 2024-11-24T09:40:44,314 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782, size=195.7 K (200357bytes) 2024-11-24T09:40:44,314 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782 2024-11-24T09:40:44,315 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782 after 0ms 2024-11-24T09:40:44,326 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:44,331 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782 took 18ms 2024-11-24T09:40:44,395 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732441243782.temp 2024-11-24T09:40:44,406 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000000001-wal.1732441243782.temp 2024-11-24T09:40:44,527 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782 so closing down 2024-11-24T09:40:44,527 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:44,527 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:44,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741867_1044 (size=200357) 2024-11-24T09:40:44,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741867_1044 (size=200357) 2024-11-24T09:40:44,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741867_1044 (size=200357) 2024-11-24T09:40:44,561 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000000001-wal.1732441243782.temp (wrote 3000 edits, skipped 0 edits in 31 ms) 2024-11-24T09:40:44,563 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000000001-wal.1732441243782.temp to hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003000 2024-11-24T09:40:44,564 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 194 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-11-24T09:40:44,564 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782, journal: Splitting hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782, size=195.7 K (200357bytes) at 1732441244314Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000000001-wal.1732441243782.temp at 1732441244406 (+92 ms)Split 1024 edits, skipped 0 edits. at 1732441244432 (+26 ms)Split 2048 edits, skipped 0 edits. at 1732441244485 (+53 ms)Finishing writing output for hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782 so closing down at 1732441244527 (+42 ms)3 split writer threads finished at 1732441244527Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000000001-wal.1732441243782.temp (wrote 3000 edits, skipped 0 edits in 31 ms) at 1732441244561 (+34 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000000001-wal.1732441243782.temp to hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003000 at 1732441244563 (+2 ms)Processed 3000 edits across 1 Regions in 194 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1732441244564 (+1 ms) 2024-11-24T09:40:44,568 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441243782 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441243782 2024-11-24T09:40:44,569 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003000 2024-11-24T09:40:44,570 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:44,575 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:44,604 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576, exclude list is [], retry=0 2024-11-24T09:40:44,620 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:44,620 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:44,620 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:44,663 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576 2024-11-24T09:40:44,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:45,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741868_1045 (size=200484) 2024-11-24T09:40:45,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741868_1045 (size=200484) 2024-11-24T09:40:45,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741868_1045 (size=200484) 2024-11-24T09:40:45,055 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576, size=195.8 K (200484bytes) 2024-11-24T09:40:45,056 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576 2024-11-24T09:40:45,056 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576 after 0ms 2024-11-24T09:40:45,060 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:45,062 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576 took 6ms 2024-11-24T09:40:45,091 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732441244576.temp 2024-11-24T09:40:45,097 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003001-wal.1732441244576.temp 2024-11-24T09:40:45,131 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576 so closing down 2024-11-24T09:40:45,131 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:45,132 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:45,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741869_1046 (size=200484) 2024-11-24T09:40:45,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741869_1046 (size=200484) 2024-11-24T09:40:45,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741869_1046 (size=200484) 2024-11-24T09:40:45,141 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003001-wal.1732441244576.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-11-24T09:40:45,147 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003001-wal.1732441244576.temp to hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000006000 2024-11-24T09:40:45,147 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-11-24T09:40:45,148 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576, journal: Splitting hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576, size=195.8 K (200484bytes) at 1732441245056Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003001-wal.1732441244576.temp at 1732441245097 (+41 ms)Split 1024 edits, skipped 0 edits. at 1732441245097Split 2048 edits, skipped 0 edits. at 1732441245112 (+15 ms)Finishing writing output for hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576 so closing down at 1732441245131 (+19 ms)3 split writer threads finished at 1732441245132 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003001-wal.1732441244576.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1732441245141 (+9 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003001-wal.1732441244576.temp to hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000006000 at 1732441245147 (+6 ms)Processed 3000 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1732441245147 2024-11-24T09:40:45,154 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441244576 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441244576 2024-11-24T09:40:45,155 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000006000 2024-11-24T09:40:45,156 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:45,159 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/test2727-manual,16010,1732441243466, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:45,179 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441245159, exclude list is [], retry=0 2024-11-24T09:40:45,183 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:45,183 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:45,184 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:45,251 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732441243466/wal.1732441245159 2024-11-24T09:40:45,266 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:45,267 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b512189a5cb637b9f79cf1f7b8015a0e, NAME => 'test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:45,267 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:45,268 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,268 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,283 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,285 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b512189a5cb637b9f79cf1f7b8015a0e columnFamilyName a 2024-11-24T09:40:45,285 DEBUG [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:45,286 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(327): Store=b512189a5cb637b9f79cf1f7b8015a0e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:45,286 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,287 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b512189a5cb637b9f79cf1f7b8015a0e columnFamilyName b 2024-11-24T09:40:45,288 DEBUG [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:45,288 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(327): Store=b512189a5cb637b9f79cf1f7b8015a0e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:45,288 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,291 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b512189a5cb637b9f79cf1f7b8015a0e columnFamilyName c 2024-11-24T09:40:45,291 DEBUG [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:45,293 INFO [StoreOpener-b512189a5cb637b9f79cf1f7b8015a0e-1 {}] regionserver.HStore(327): Store=b512189a5cb637b9f79cf1f7b8015a0e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:45,293 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,294 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,310 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003000 2024-11-24T09:40:45,321 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:45,433 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003000 2024-11-24T09:40:45,436 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000006000 2024-11-24T09:40:45,451 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:45,553 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000006000 2024-11-24T09:40:45,553 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b512189a5cb637b9f79cf1f7b8015a0e 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T09:40:45,605 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/a/18b117231dd94467979f0aec046143a9 is 41, key is test2727/a:100/1732441244676/Put/seqid=0 2024-11-24T09:40:45,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741871_1048 (size=84227) 2024-11-24T09:40:45,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741871_1048 (size=84227) 2024-11-24T09:40:45,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741871_1048 (size=84227) 2024-11-24T09:40:45,655 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/a/18b117231dd94467979f0aec046143a9 2024-11-24T09:40:45,710 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/b/de9f1ce82b2b474eb0297231e6fa3edf is 41, key is test2727/b:100/1732441244869/Put/seqid=0 2024-11-24T09:40:45,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741872_1049 (size=84609) 2024-11-24T09:40:45,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741872_1049 (size=84609) 2024-11-24T09:40:45,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741872_1049 (size=84609) 2024-11-24T09:40:45,766 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/b/de9f1ce82b2b474eb0297231e6fa3edf 2024-11-24T09:40:45,833 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/c/3edba477717f4588bc710fb066c6fcf3 is 41, key is test2727/c:100/1732441244943/Put/seqid=0 2024-11-24T09:40:45,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741873_1050 (size=84609) 2024-11-24T09:40:45,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741873_1050 (size=84609) 2024-11-24T09:40:45,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741873_1050 (size=84609) 2024-11-24T09:40:45,880 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/c/3edba477717f4588bc710fb066c6fcf3 2024-11-24T09:40:45,894 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/a/18b117231dd94467979f0aec046143a9 as hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/a/18b117231dd94467979f0aec046143a9 2024-11-24T09:40:45,905 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/a/18b117231dd94467979f0aec046143a9, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T09:40:45,907 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/b/de9f1ce82b2b474eb0297231e6fa3edf as hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/b/de9f1ce82b2b474eb0297231e6fa3edf 2024-11-24T09:40:45,914 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/b/de9f1ce82b2b474eb0297231e6fa3edf, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T09:40:45,916 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/.tmp/c/3edba477717f4588bc710fb066c6fcf3 as hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/c/3edba477717f4588bc710fb066c6fcf3 2024-11-24T09:40:45,926 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/c/3edba477717f4588bc710fb066c6fcf3, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T09:40:45,927 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for b512189a5cb637b9f79cf1f7b8015a0e in 374ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T09:40:45,928 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000003000 2024-11-24T09:40:45,929 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/0000000000000006000 2024-11-24T09:40:45,931 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,931 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,932 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:45,936 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b512189a5cb637b9f79cf1f7b8015a0e 2024-11-24T09:40:45,940 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/test2727/b512189a5cb637b9f79cf1f7b8015a0e/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T09:40:45,942 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b512189a5cb637b9f79cf1f7b8015a0e; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58994335, jitterRate=-0.12091590464115143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:45,942 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b512189a5cb637b9f79cf1f7b8015a0e: Writing region info on filesystem at 1732441245268Initializing all the Stores at 1732441245270 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441245270Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441245283 (+13 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441245283Obtaining lock to block concurrent updates at 1732441245553 (+270 ms)Preparing flush snapshotting stores in b512189a5cb637b9f79cf1f7b8015a0e at 1732441245553Finished memstore snapshotting test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732441245554 (+1 ms)Flushing stores of test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. at 1732441245554Flushing b512189a5cb637b9f79cf1f7b8015a0e/a: creating writer at 1732441245554Flushing b512189a5cb637b9f79cf1f7b8015a0e/a: appending metadata at 1732441245603 (+49 ms)Flushing b512189a5cb637b9f79cf1f7b8015a0e/a: closing flushed file at 1732441245604 (+1 ms)Flushing b512189a5cb637b9f79cf1f7b8015a0e/b: creating writer at 1732441245667 (+63 ms)Flushing b512189a5cb637b9f79cf1f7b8015a0e/b: appending metadata at 1732441245706 (+39 ms)Flushing b512189a5cb637b9f79cf1f7b8015a0e/b: closing flushed file at 1732441245706Flushing b512189a5cb637b9f79cf1f7b8015a0e/c: creating writer at 1732441245776 (+70 ms)Flushing b512189a5cb637b9f79cf1f7b8015a0e/c: appending metadata at 1732441245831 (+55 ms)Flushing b512189a5cb637b9f79cf1f7b8015a0e/c: closing flushed file at 1732441245831Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33e82289: reopening flushed file at 1732441245892 (+61 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7be414df: reopening flushed file at 1732441245906 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@320fe1b6: reopening flushed file at 1732441245914 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for b512189a5cb637b9f79cf1f7b8015a0e in 374ms, sequenceid=6000, compaction requested=false; wal=null at 1732441245927 (+13 ms)Cleaning up temporary data from old regions at 1732441245931 (+4 ms)Region opened successfully at 1732441245942 (+11 ms) 2024-11-24T09:40:45,944 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T09:40:45,944 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b512189a5cb637b9f79cf1f7b8015a0e, disabling compactions & flushes 2024-11-24T09:40:45,944 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. 2024-11-24T09:40:45,944 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. 2024-11-24T09:40:45,944 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. after waiting 0 ms 2024-11-24T09:40:45,944 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. 2024-11-24T09:40:45,956 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732441243468.b512189a5cb637b9f79cf1f7b8015a0e. 2024-11-24T09:40:45,956 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b512189a5cb637b9f79cf1f7b8015a0e: Waiting for close lock at 1732441245944Disabling compacts and flushes for region at 1732441245944Disabling writes for close at 1732441245944Writing region close event to WAL at 1732441245956 (+12 ms)Closed at 1732441245956 2024-11-24T09:40:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741870_1047 (size=93) 2024-11-24T09:40:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741870_1047 (size=93) 2024-11-24T09:40:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741870_1047 (size=93) 2024-11-24T09:40:45,979 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:45,979 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441245159) 2024-11-24T09:40:45,996 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=401 (was 401), OpenFileDescriptor=923 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=571 (was 571), ProcessCount=11 (was 11), AvailableMemoryMB=7832 (was 7984) 2024-11-24T09:40:46,010 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=399, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=571, ProcessCount=11, AvailableMemoryMB=7832 2024-11-24T09:40:46,058 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:46,077 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:46,078 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732441246078 2024-11-24T09:40:46,124 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078 2024-11-24T09:40:46,133 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:46,141 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97423ad3e4e1d36ebe5d81cc4df2400e, NAME => 'testSequentialEditLogSeqNum,,1732441246059.97423ad3e4e1d36ebe5d81cc4df2400e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:46,142 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732441246059.97423ad3e4e1d36ebe5d81cc4df2400e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:46,142 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,142 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,144 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e doesn't exist for region: 97423ad3e4e1d36ebe5d81cc4df2400e on table testSequentialEditLogSeqNum 2024-11-24T09:40:46,145 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 97423ad3e4e1d36ebe5d81cc4df2400e on table testSequentialEditLogSeqNum 2024-11-24T09:40:46,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741875_1052 (size=62) 2024-11-24T09:40:46,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741875_1052 (size=62) 2024-11-24T09:40:46,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741875_1052 (size=62) 2024-11-24T09:40:46,192 INFO [StoreOpener-97423ad3e4e1d36ebe5d81cc4df2400e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,196 INFO [StoreOpener-97423ad3e4e1d36ebe5d81cc4df2400e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97423ad3e4e1d36ebe5d81cc4df2400e columnFamilyName a 2024-11-24T09:40:46,196 DEBUG [StoreOpener-97423ad3e4e1d36ebe5d81cc4df2400e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:46,197 INFO [StoreOpener-97423ad3e4e1d36ebe5d81cc4df2400e-1 {}] regionserver.HStore(327): Store=97423ad3e4e1d36ebe5d81cc4df2400e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:46,197 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,198 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,198 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,199 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,199 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,200 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97423ad3e4e1d36ebe5d81cc4df2400e 2024-11-24T09:40:46,214 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:46,214 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97423ad3e4e1d36ebe5d81cc4df2400e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66629451, jitterRate=-0.007143810391426086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:40:46,215 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97423ad3e4e1d36ebe5d81cc4df2400e: Writing region info on filesystem at 1732441246142Initializing all the Stores at 1732441246192 (+50 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441246192Cleaning up temporary data from old regions at 1732441246199 (+7 ms)Region opened successfully at 1732441246215 (+16 ms) 2024-11-24T09:40:46,234 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97423ad3e4e1d36ebe5d81cc4df2400e 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T09:40:46,269 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/.tmp/a/3b416391e6384f5b84ed0b00c58cef74 is 81, key is testSequentialEditLogSeqNum/a:x0/1732441246215/Put/seqid=0 2024-11-24T09:40:46,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741876_1053 (size=5833) 2024-11-24T09:40:46,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741876_1053 (size=5833) 2024-11-24T09:40:46,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741876_1053 (size=5833) 2024-11-24T09:40:46,304 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/.tmp/a/3b416391e6384f5b84ed0b00c58cef74 2024-11-24T09:40:46,323 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/.tmp/a/3b416391e6384f5b84ed0b00c58cef74 as hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/a/3b416391e6384f5b84ed0b00c58cef74 2024-11-24T09:40:46,342 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/a/3b416391e6384f5b84ed0b00c58cef74, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T09:40:46,344 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 97423ad3e4e1d36ebe5d81cc4df2400e in 110ms, sequenceid=13, compaction requested=false 2024-11-24T09:40:46,344 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 97423ad3e4e1d36ebe5d81cc4df2400e: 2024-11-24T09:40:46,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:40:46,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:40:46,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:40:46,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:40:46,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T09:40:46,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741874_1051 (size=1616) 2024-11-24T09:40:46,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741874_1051 (size=1616) 2024-11-24T09:40:46,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741874_1051 (size=1616) 2024-11-24T09:40:46,395 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078, size=1.6 K (1616bytes) 2024-11-24T09:40:46,395 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078 2024-11-24T09:40:46,396 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078 after 1ms 2024-11-24T09:40:46,407 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:46,407 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078 took 12ms 2024-11-24T09:40:46,414 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078 so closing down 2024-11-24T09:40:46,414 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:46,417 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441246078.temp 2024-11-24T09:40:46,419 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/0000000000000000003-wal.1732441246078.temp 2024-11-24T09:40:46,420 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:46,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741877_1054 (size=1175) 2024-11-24T09:40:46,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741877_1054 (size=1175) 2024-11-24T09:40:46,428 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/0000000000000000003-wal.1732441246078.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T09:40:46,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741877_1054 (size=1175) 2024-11-24T09:40:46,430 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/0000000000000000003-wal.1732441246078.temp to hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/0000000000000000020 2024-11-24T09:40:46,430 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-11-24T09:40:46,430 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078, journal: Splitting hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078, size=1.6 K (1616bytes) at 1732441246395Finishing writing output for hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078 so closing down at 1732441246414 (+19 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/0000000000000000003-wal.1732441246078.temp at 1732441246419 (+5 ms)3 split writer threads finished at 1732441246420 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/0000000000000000003-wal.1732441246078.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732441246428 (+8 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/0000000000000000003-wal.1732441246078.temp to hdfs://localhost:43013/hbase/data/default/testSequentialEditLogSeqNum/97423ad3e4e1d36ebe5d81cc4df2400e/recovered.edits/0000000000000000020 at 1732441246430 (+2 ms)Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:43013/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732441246057/wal.1732441246078, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1732441246430 2024-11-24T09:40:46,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=404 (was 399) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:45528 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:58474 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=571 (was 571), ProcessCount=11 (was 11), AvailableMemoryMB=7829 (was 7832) 2024-11-24T09:40:46,458 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=404, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=571, ProcessCount=11, AvailableMemoryMB=7828 2024-11-24T09:40:46,475 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:46,478 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:46,519 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:46,536 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-14082939, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-14082939, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:46,556 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-14082939/hregion-14082939.1732441246536, exclude list is [], retry=0 2024-11-24T09:40:46,561 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:46,563 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:46,587 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:46,597 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-14082939/hregion-14082939.1732441246536 2024-11-24T09:40:46,599 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:40:46,599 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ee8068b082717ffce6a19a606b5148a0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:40:46,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741879_1056 (size=70) 2024-11-24T09:40:46,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741879_1056 (size=70) 2024-11-24T09:40:46,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741879_1056 (size=70) 2024-11-24T09:40:46,636 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:46,644 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,648 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName a 2024-11-24T09:40:46,648 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:46,649 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:46,649 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,651 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName b 2024-11-24T09:40:46,651 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:46,652 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:46,652 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,659 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName c 2024-11-24T09:40:46,659 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:46,660 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:46,661 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,665 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,665 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,665 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:46,667 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,675 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:46,676 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee8068b082717ffce6a19a606b5148a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74760204, jitterRate=0.11401385068893433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:46,677 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee8068b082717ffce6a19a606b5148a0: Writing region info on filesystem at 1732441246636Initializing all the Stores at 1732441246643 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441246643Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441246643Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441246643Cleaning up temporary data from old regions at 1732441246665 (+22 ms)Region opened successfully at 1732441246677 (+12 ms) 2024-11-24T09:40:46,678 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee8068b082717ffce6a19a606b5148a0, disabling compactions & flushes 2024-11-24T09:40:46,678 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. 2024-11-24T09:40:46,678 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. 2024-11-24T09:40:46,678 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. after waiting 0 ms 2024-11-24T09:40:46,678 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. 2024-11-24T09:40:46,687 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. 2024-11-24T09:40:46,687 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee8068b082717ffce6a19a606b5148a0: Waiting for close lock at 1732441246677Disabling compacts and flushes for region at 1732441246677Disabling writes for close at 1732441246678 (+1 ms)Writing region close event to WAL at 1732441246687 (+9 ms)Closed at 1732441246687 2024-11-24T09:40:46,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741878_1055 (size=93) 2024-11-24T09:40:46,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741878_1055 (size=93) 2024-11-24T09:40:46,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741878_1055 (size=93) 2024-11-24T09:40:46,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:46,696 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-14082939:(num 1732441246536) 2024-11-24T09:40:46,696 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:46,702 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:46,721 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702, exclude list is [], retry=0 2024-11-24T09:40:46,724 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:46,725 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:46,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:46,740 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 2024-11-24T09:40:46,740 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:46,740 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee8068b082717ffce6a19a606b5148a0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:46,741 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:46,741 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,741 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,744 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,745 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName a 2024-11-24T09:40:46,745 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:46,746 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:46,746 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,747 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName b 2024-11-24T09:40:46,747 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:46,748 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:46,749 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,750 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName c 2024-11-24T09:40:46,750 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:46,751 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:46,751 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,752 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,753 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,755 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:46,758 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:46,763 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee8068b082717ffce6a19a606b5148a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69569501, jitterRate=0.036666348576545715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:46,764 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee8068b082717ffce6a19a606b5148a0: Writing region info on filesystem at 1732441246741Initializing all the Stores at 1732441246743 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441246744 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441246744Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441246744Cleaning up temporary data from old regions at 1732441246754 (+10 ms)Region opened successfully at 1732441246764 (+10 ms) 2024-11-24T09:40:46,771 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732441246769/Put/seqid=0 2024-11-24T09:40:46,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741881_1058 (size=4826) 2024-11-24T09:40:46,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741881_1058 (size=4826) 2024-11-24T09:40:46,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741881_1058 (size=4826) 2024-11-24T09:40:46,818 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43013/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in ee8068b082717ffce6a19a606b5148a0/a 2024-11-24T09:40:46,829 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T09:40:46,831 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T09:40:46,832 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ee8068b082717ffce6a19a606b5148a0: 2024-11-24T09:40:46,835 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/a/5c8bae09f09b4b6ba336b83ae2867ea7_SeqId_3_ 2024-11-24T09:40:46,836 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43013/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into ee8068b082717ffce6a19a606b5148a0/a as hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/a/5c8bae09f09b4b6ba336b83ae2867ea7_SeqId_3_ - updating store file list. 2024-11-24T09:40:46,844 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5c8bae09f09b4b6ba336b83ae2867ea7_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T09:40:46,845 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/a/5c8bae09f09b4b6ba336b83ae2867ea7_SeqId_3_ into ee8068b082717ffce6a19a606b5148a0/a 2024-11-24T09:40:46,845 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43013/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into ee8068b082717ffce6a19a606b5148a0/a (new location: hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/a/5c8bae09f09b4b6ba336b83ae2867ea7_SeqId_3_) 2024-11-24T09:40:47,007 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702, size=0 (0bytes) 2024-11-24T09:40:47,007 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 might be still open, length is 0 2024-11-24T09:40:47,008 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 2024-11-24T09:40:47,008 WARN [IPC Server handler 3 on default port 43013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T09:40:47,012 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 after 4ms 2024-11-24T09:40:48,921 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:58660 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45495:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58660 dst: /127.0.0.1:45495 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45495 remote=/127.0.0.1:58660]. Total timeout mills is 60000, 57956 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:48,921 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:46836 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43599:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46836 dst: /127.0.0.1:43599 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:48,922 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:45628 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45595:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45628 dst: /127.0.0.1:45595 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:48,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741880_1059 (size=434) 2024-11-24T09:40:48,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741880_1059 (size=434) 2024-11-24T09:40:51,013 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 after 4005ms 2024-11-24T09:40:51,017 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:51,017 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 took 4011ms 2024-11-24T09:40:51,019 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702; continuing. 2024-11-24T09:40:51,019 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 so closing down 2024-11-24T09:40:51,019 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:51,023 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732441246702.temp 2024-11-24T09:40:51,024 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005-wal.1732441246702.temp 2024-11-24T09:40:51,025 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741882_1060 (size=236) 2024-11-24T09:40:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741882_1060 (size=236) 2024-11-24T09:40:51,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741882_1060 (size=236) 2024-11-24T09:40:51,032 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005-wal.1732441246702.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T09:40:51,034 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005-wal.1732441246702.temp to hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005 2024-11-24T09:40:51,034 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T09:40:51,034 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702, journal: Splitting hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702, size=0 (0bytes) at 1732441247007Finishing writing output for hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 so closing down at 1732441251019 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005-wal.1732441246702.temp at 1732441251024 (+5 ms)3 split writer threads finished at 1732441251025 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005-wal.1732441246702.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732441251032 (+7 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005-wal.1732441246702.temp to hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005 at 1732441251034 (+2 ms)Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702, size=0, length=0, corrupted=false, cancelled=false at 1732441251034 2024-11-24T09:40:51,036 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441246702 2024-11-24T09:40:51,037 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005 2024-11-24T09:40:51,037 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:51,039 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:51,054 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441251039, exclude list is [], retry=0 2024-11-24T09:40:51,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:51,057 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:51,057 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:51,059 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441251039 2024-11-24T09:40:51,061 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:51,062 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee8068b082717ffce6a19a606b5148a0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:51,062 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:51,062 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,062 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,067 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,068 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName a 2024-11-24T09:40:51,068 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:51,086 DEBUG [StoreFileOpener-ee8068b082717ffce6a19a606b5148a0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5c8bae09f09b4b6ba336b83ae2867ea7_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T09:40:51,086 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/a/5c8bae09f09b4b6ba336b83ae2867ea7_SeqId_3_ 2024-11-24T09:40:51,086 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:51,086 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,087 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName b 2024-11-24T09:40:51,088 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:51,088 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:51,088 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,089 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee8068b082717ffce6a19a606b5148a0 columnFamilyName c 2024-11-24T09:40:51,089 DEBUG [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:51,090 INFO [StoreOpener-ee8068b082717ffce6a19a606b5148a0-1 {}] regionserver.HStore(327): Store=ee8068b082717ffce6a19a606b5148a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:51,090 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,091 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,094 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,095 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005 2024-11-24T09:40:51,099 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:51,101 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005 2024-11-24T09:40:51,101 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ee8068b082717ffce6a19a606b5148a0 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T09:40:51,120 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/.tmp/a/8170b859f0634a129b434caf9ca60313 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732441246875/Put/seqid=0 2024-11-24T09:40:51,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741884_1062 (size=5149) 2024-11-24T09:40:51,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741884_1062 (size=5149) 2024-11-24T09:40:51,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741884_1062 (size=5149) 2024-11-24T09:40:51,145 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/.tmp/a/8170b859f0634a129b434caf9ca60313 2024-11-24T09:40:51,154 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/.tmp/a/8170b859f0634a129b434caf9ca60313 as hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/a/8170b859f0634a129b434caf9ca60313 2024-11-24T09:40:51,162 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/a/8170b859f0634a129b434caf9ca60313, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T09:40:51,162 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for ee8068b082717ffce6a19a606b5148a0 in 61ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T09:40:51,164 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/0000000000000000005 2024-11-24T09:40:51,166 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,167 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,167 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:51,169 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee8068b082717ffce6a19a606b5148a0 2024-11-24T09:40:51,173 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ee8068b082717ffce6a19a606b5148a0/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T09:40:51,174 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee8068b082717ffce6a19a606b5148a0; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60521879, jitterRate=-0.0981537252664566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:51,175 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee8068b082717ffce6a19a606b5148a0: Writing region info on filesystem at 1732441251062Initializing all the Stores at 1732441251064 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441251064Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441251066 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441251066Obtaining lock to block concurrent updates at 1732441251101 (+35 ms)Preparing flush snapshotting stores in ee8068b082717ffce6a19a606b5148a0 at 1732441251101Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732441251101Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. at 1732441251101Flushing ee8068b082717ffce6a19a606b5148a0/a: creating writer at 1732441251101Flushing ee8068b082717ffce6a19a606b5148a0/a: appending metadata at 1732441251120 (+19 ms)Flushing ee8068b082717ffce6a19a606b5148a0/a: closing flushed file at 1732441251120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b139f5a: reopening flushed file at 1732441251151 (+31 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for ee8068b082717ffce6a19a606b5148a0 in 61ms, sequenceid=5, compaction requested=false; wal=null at 1732441251162 (+11 ms)Cleaning up temporary data from old regions at 1732441251167 (+5 ms)Region opened successfully at 1732441251175 (+8 ms) 2024-11-24T09:40:51,179 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee8068b082717ffce6a19a606b5148a0, disabling compactions & flushes 2024-11-24T09:40:51,179 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. 2024-11-24T09:40:51,179 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. 2024-11-24T09:40:51,179 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. after waiting 0 ms 2024-11-24T09:40:51,179 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. 2024-11-24T09:40:51,180 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732441246476.ee8068b082717ffce6a19a606b5148a0. 2024-11-24T09:40:51,180 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee8068b082717ffce6a19a606b5148a0: Waiting for close lock at 1732441251179Disabling compacts and flushes for region at 1732441251179Disabling writes for close at 1732441251179Writing region close event to WAL at 1732441251180 (+1 ms)Closed at 1732441251180 2024-11-24T09:40:51,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741883_1061 (size=93) 2024-11-24T09:40:51,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741883_1061 (size=93) 2024-11-24T09:40:51,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741883_1061 (size=93) 2024-11-24T09:40:51,192 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:51,192 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441251039) 2024-11-24T09:40:51,206 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 404) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:43013 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (373550922) connection to localhost/127.0.0.1:43013 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1980241825_22 at /127.0.0.1:46854 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1980241825_22 at /127.0.0.1:58704 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1980241825_22 at /127.0.0.1:45682 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=590 (was 571) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7799 (was 7828) 2024-11-24T09:40:51,218 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=590, ProcessCount=11, AvailableMemoryMB=7798 2024-11-24T09:40:51,234 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:51,240 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T09:40:51,245 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ba35ada4783f,43385,1732441231965 2024-11-24T09:40:51,248 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d0256bf 2024-11-24T09:40:51,249 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T09:40:51,251 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T09:40:51,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T09:40:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T09:40:51,266 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T09:40:51,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T09:40:51,270 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:51,272 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T09:40:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:40:51,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741885_1063 (size=694) 2024-11-24T09:40:51,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741885_1063 (size=694) 2024-11-24T09:40:51,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741885_1063 (size=694) 2024-11-24T09:40:51,291 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a47f41b92e82354ab3fd0381bbcca2bb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63 2024-11-24T09:40:51,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741886_1064 (size=77) 2024-11-24T09:40:51,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741886_1064 (size=77) 2024-11-24T09:40:51,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741886_1064 (size=77) 2024-11-24T09:40:51,301 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:51,302 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing a47f41b92e82354ab3fd0381bbcca2bb, disabling compactions & flushes 2024-11-24T09:40:51,302 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:51,302 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:51,302 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. after waiting 0 ms 2024-11-24T09:40:51,302 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:51,302 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:51,302 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for a47f41b92e82354ab3fd0381bbcca2bb: Waiting for close lock at 1732441251302Disabling compacts and flushes for region at 1732441251302Disabling writes for close at 1732441251302Writing region close event to WAL at 1732441251302Closed at 1732441251302 2024-11-24T09:40:51,304 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T09:40:51,309 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732441251304"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732441251304"}]},"ts":"1732441251304"} 2024-11-24T09:40:51,313 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T09:40:51,315 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T09:40:51,318 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732441251315"}]},"ts":"1732441251315"} 2024-11-24T09:40:51,321 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T09:40:51,322 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {ba35ada4783f=0} racks are {/default-rack=0} 2024-11-24T09:40:51,323 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T09:40:51,323 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T09:40:51,323 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T09:40:51,323 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T09:40:51,323 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T09:40:51,323 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T09:40:51,323 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T09:40:51,323 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T09:40:51,323 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T09:40:51,323 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T09:40:51,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN}] 2024-11-24T09:40:51,327 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN 2024-11-24T09:40:51,328 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN; state=OFFLINE, location=ba35ada4783f,42723,1732441233325; forceNewPlan=false, retain=false 2024-11-24T09:40:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:40:51,481 INFO [ba35ada4783f:43385 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T09:40:51,482 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=OPENING, regionLocation=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:51,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN because future has completed 2024-11-24T09:40:51,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325}] 2024-11-24T09:40:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:40:51,650 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:51,650 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a47f41b92e82354ab3fd0381bbcca2bb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:51,651 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,651 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:51,651 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,651 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,655 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,656 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf1 2024-11-24T09:40:51,656 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:51,657 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:51,657 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,659 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf2 2024-11-24T09:40:51,659 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:51,660 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:51,660 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,662 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,663 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,663 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,663 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,665 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T09:40:51,667 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,683 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:51,684 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a47f41b92e82354ab3fd0381bbcca2bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69134393, jitterRate=0.03018273413181305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T09:40:51,684 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:51,685 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a47f41b92e82354ab3fd0381bbcca2bb: Running coprocessor pre-open hook at 1732441251651Writing region info on filesystem at 1732441251651Initializing all the Stores at 1732441251652 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441251652Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441251654 (+2 ms)Cleaning up temporary data from old regions at 1732441251663 (+9 ms)Running coprocessor post-open hooks at 1732441251684 (+21 ms)Region opened successfully at 1732441251685 (+1 ms) 2024-11-24T09:40:51,689 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., pid=6, masterSystemTime=1732441251640 2024-11-24T09:40:51,695 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=OPEN, openSeqNum=2, regionLocation=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:51,695 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:51,695 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:51,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325 because future has completed 2024-11-24T09:40:51,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T09:40:51,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325 in 216 msec 2024-11-24T09:40:51,719 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T09:40:51,719 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732441251719"}]},"ts":"1732441251719"} 2024-11-24T09:40:51,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T09:40:51,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN in 386 msec 2024-11-24T09:40:51,724 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T09:40:51,727 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T09:40:51,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 469 msec 2024-11-24T09:40:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:40:51,904 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T09:40:51,904 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T09:40:51,905 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T09:40:51,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T09:40:51,916 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T09:40:51,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T09:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=2] 2024-11-24T09:40:51,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=a47f41b92e82354ab3fd0381bbcca2bb, source=ba35ada4783f,42723,1732441233325, destination=ba35ada4783f,36811,1732441233150, warming up region on ba35ada4783f,36811,1732441233150 2024-11-24T09:40:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:40:51,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=a47f41b92e82354ab3fd0381bbcca2bb, source=ba35ada4783f,42723,1732441233325, destination=ba35ada4783f,36811,1732441233150, running balancer 2024-11-24T09:40:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE 2024-11-24T09:40:51,969 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE 2024-11-24T09:40:51,973 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=CLOSING, regionLocation=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:51,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE because future has completed 2024-11-24T09:40:51,981 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T09:40:51,981 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=ba35ada4783f,42723,1732441233325, table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T09:40:51,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325}] 2024-11-24T09:40:51,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:40:51,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(7855): Warmup {ENCODED => a47f41b92e82354ab3fd0381bbcca2bb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:51,998 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,000 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf1 2024-11-24T09:40:52,000 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:52,001 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:52,001 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,003 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf2 2024-11-24T09:40:52,003 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:52,004 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(1722): Closing a47f41b92e82354ab3fd0381bbcca2bb, disabling compactions & flushes 2024-11-24T09:40:52,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. after waiting 0 ms 2024-11-24T09:40:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] regionserver.HRegion(1676): Region close journal for a47f41b92e82354ab3fd0381bbcca2bb: Waiting for close lock at 1732441252004Disabling compacts and flushes for region at 1732441252004Disabling writes for close at 1732441252004Writing region close event to WAL at 1732441252019 (+15 ms)Closed at 1732441252019 2024-11-24T09:40:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T09:40:52,141 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,142 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T09:40:52,143 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing a47f41b92e82354ab3fd0381bbcca2bb, disabling compactions & flushes 2024-11-24T09:40:52,143 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,143 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,143 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. after waiting 0 ms 2024-11-24T09:40:52,143 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,143 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing a47f41b92e82354ab3fd0381bbcca2bb 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T09:40:52,166 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf1/e69dcfaf066c4daf9f1195642ec0ee41 is 35, key is r1/cf1:q/1732441251942/Put/seqid=0 2024-11-24T09:40:52,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741887_1065 (size=4783) 2024-11-24T09:40:52,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741887_1065 (size=4783) 2024-11-24T09:40:52,225 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf1/e69dcfaf066c4daf9f1195642ec0ee41 2024-11-24T09:40:52,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741887_1065 (size=4783) 2024-11-24T09:40:52,234 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf1/e69dcfaf066c4daf9f1195642ec0ee41 as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/e69dcfaf066c4daf9f1195642ec0ee41 2024-11-24T09:40:52,244 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/e69dcfaf066c4daf9f1195642ec0ee41, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T09:40:52,246 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for a47f41b92e82354ab3fd0381bbcca2bb in 103ms, sequenceid=5, compaction requested=false 2024-11-24T09:40:52,247 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T09:40:52,258 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T09:40:52,261 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,261 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for a47f41b92e82354ab3fd0381bbcca2bb: Waiting for close lock at 1732441252142Running coprocessor pre-close hooks at 1732441252143 (+1 ms)Disabling compacts and flushes for region at 1732441252143Disabling writes for close at 1732441252143Obtaining lock to block concurrent updates at 1732441252143Preparing flush snapshotting stores in a47f41b92e82354ab3fd0381bbcca2bb at 1732441252143Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732441252143Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. at 1732441252145 (+2 ms)Flushing a47f41b92e82354ab3fd0381bbcca2bb/cf1: creating writer at 1732441252145Flushing a47f41b92e82354ab3fd0381bbcca2bb/cf1: appending metadata at 1732441252166 (+21 ms)Flushing a47f41b92e82354ab3fd0381bbcca2bb/cf1: closing flushed file at 1732441252166Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2becf7a8: reopening flushed file at 1732441252233 (+67 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for a47f41b92e82354ab3fd0381bbcca2bb in 103ms, sequenceid=5, compaction requested=false at 1732441252246 (+13 ms)Writing region close event to WAL at 1732441252250 (+4 ms)Running coprocessor post-close hooks at 1732441252259 (+9 ms)Closed at 1732441252261 (+2 ms) 2024-11-24T09:40:52,261 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding a47f41b92e82354ab3fd0381bbcca2bb move to ba35ada4783f,36811,1732441233150 record at close sequenceid=5 2024-11-24T09:40:52,266 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,267 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=CLOSED 2024-11-24T09:40:52,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325 because future has completed 2024-11-24T09:40:52,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T09:40:52,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325 in 298 msec 2024-11-24T09:40:52,286 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE; state=CLOSED, location=ba35ada4783f,36811,1732441233150; forceNewPlan=false, retain=false 2024-11-24T09:40:52,426 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T09:40:52,426 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T09:40:52,428 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T09:40:52,428 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T09:40:52,437 INFO [ba35ada4783f:43385 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T09:40:52,437 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=OPENING, regionLocation=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:52,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE because future has completed 2024-11-24T09:40:52,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150}] 2024-11-24T09:40:52,601 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,601 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => a47f41b92e82354ab3fd0381bbcca2bb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:52,602 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,602 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:52,602 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,602 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,608 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,610 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf1 2024-11-24T09:40:52,610 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:52,619 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/e69dcfaf066c4daf9f1195642ec0ee41 2024-11-24T09:40:52,619 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:52,619 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,620 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf2 2024-11-24T09:40:52,620 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:52,621 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:52,621 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,622 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,623 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,624 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,624 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,624 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T09:40:52,629 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,630 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened a47f41b92e82354ab3fd0381bbcca2bb; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60592836, jitterRate=-0.09709638357162476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T09:40:52,630 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:52,631 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for a47f41b92e82354ab3fd0381bbcca2bb: Running coprocessor pre-open hook at 1732441252602Writing region info on filesystem at 1732441252602Initializing all the Stores at 1732441252603 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441252603Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441252608 (+5 ms)Cleaning up temporary data from old regions at 1732441252624 (+16 ms)Running coprocessor post-open hooks at 1732441252630 (+6 ms)Region opened successfully at 1732441252631 (+1 ms) 2024-11-24T09:40:52,632 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., pid=9, masterSystemTime=1732441252597 2024-11-24T09:40:52,635 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,635 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,635 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=OPEN, openSeqNum=9, regionLocation=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:52,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150 because future has completed 2024-11-24T09:40:52,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T09:40:52,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150 in 198 msec 2024-11-24T09:40:52,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE in 681 msec 2024-11-24T09:40:52,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:40:52,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:40:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:53408 deadline: 1732441312681, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=36811 startCode=1732441233150. As of locationSeqNum=5. 2024-11-24T09:40:52,689 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=36811 startCode=1732441233150. As of locationSeqNum=5. 2024-11-24T09:40:52,690 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=36811 startCode=1732441233150. As of locationSeqNum=5. 2024-11-24T09:40:52,690 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,36811,1732441233150, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=36811 startCode=1732441233150. As of locationSeqNum=5. 2024-11-24T09:40:52,803 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:40:52,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:40:52,814 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a47f41b92e82354ab3fd0381bbcca2bb 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T09:40:52,833 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf1/2712e6288b4c462a98927f61046a782a is 29, key is r1/cf1:/1732441252806/DeleteFamily/seqid=0 2024-11-24T09:40:52,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741888_1066 (size=4906) 2024-11-24T09:40:52,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741888_1066 (size=4906) 2024-11-24T09:40:52,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741888_1066 (size=4906) 2024-11-24T09:40:52,858 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf1/2712e6288b4c462a98927f61046a782a 2024-11-24T09:40:52,866 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2712e6288b4c462a98927f61046a782a 2024-11-24T09:40:52,889 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf2/1e45189b6e4049218c48a4367d2c5670 is 29, key is r1/cf2:/1732441252806/DeleteFamily/seqid=0 2024-11-24T09:40:52,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741889_1067 (size=4906) 2024-11-24T09:40:52,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741889_1067 (size=4906) 2024-11-24T09:40:52,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741889_1067 (size=4906) 2024-11-24T09:40:52,912 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf2/1e45189b6e4049218c48a4367d2c5670 2024-11-24T09:40:52,921 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e45189b6e4049218c48a4367d2c5670 2024-11-24T09:40:52,922 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf1/2712e6288b4c462a98927f61046a782a as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/2712e6288b4c462a98927f61046a782a 2024-11-24T09:40:52,931 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2712e6288b4c462a98927f61046a782a 2024-11-24T09:40:52,931 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/2712e6288b4c462a98927f61046a782a, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T09:40:52,933 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf2/1e45189b6e4049218c48a4367d2c5670 as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/1e45189b6e4049218c48a4367d2c5670 2024-11-24T09:40:52,941 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e45189b6e4049218c48a4367d2c5670 2024-11-24T09:40:52,941 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/1e45189b6e4049218c48a4367d2c5670, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T09:40:52,943 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for a47f41b92e82354ab3fd0381bbcca2bb in 128ms, sequenceid=12, compaction requested=false 2024-11-24T09:40:52,943 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a47f41b92e82354ab3fd0381bbcca2bb: 2024-11-24T09:40:52,946 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T09:40:52,947 DEBUG [Time-limited test {}] regionserver.HStore(1541): a47f41b92e82354ab3fd0381bbcca2bb/cf1 is initiating major compaction (all files) 2024-11-24T09:40:52,947 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:40:52,947 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:52,947 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of a47f41b92e82354ab3fd0381bbcca2bb/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:52,948 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/e69dcfaf066c4daf9f1195642ec0ee41, hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/2712e6288b4c462a98927f61046a782a] into tmpdir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp, totalSize=9.5 K 2024-11-24T09:40:52,949 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e69dcfaf066c4daf9f1195642ec0ee41, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732441251942 2024-11-24T09:40:52,950 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2712e6288b4c462a98927f61046a782a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T09:40:52,963 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): a47f41b92e82354ab3fd0381bbcca2bb#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T09:40:52,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741890_1068 (size=4626) 2024-11-24T09:40:52,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741890_1068 (size=4626) 2024-11-24T09:40:52,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741890_1068 (size=4626) 2024-11-24T09:40:52,991 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf1/ff9108722adf4af1be1c05334e2a9394 as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/ff9108722adf4af1be1c05334e2a9394 2024-11-24T09:40:53,016 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in a47f41b92e82354ab3fd0381bbcca2bb/cf1 of a47f41b92e82354ab3fd0381bbcca2bb into ff9108722adf4af1be1c05334e2a9394(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T09:40:53,016 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for a47f41b92e82354ab3fd0381bbcca2bb: 2024-11-24T09:40:53,016 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T09:40:53,016 DEBUG [Time-limited test {}] regionserver.HStore(1541): a47f41b92e82354ab3fd0381bbcca2bb/cf2 is initiating major compaction (all files) 2024-11-24T09:40:53,017 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:40:53,017 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:40:53,017 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of a47f41b92e82354ab3fd0381bbcca2bb/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,017 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/1e45189b6e4049218c48a4367d2c5670] into tmpdir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp, totalSize=4.8 K 2024-11-24T09:40:53,018 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1e45189b6e4049218c48a4367d2c5670, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T09:40:53,025 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): a47f41b92e82354ab3fd0381bbcca2bb#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T09:40:53,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741891_1069 (size=4592) 2024-11-24T09:40:53,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741891_1069 (size=4592) 2024-11-24T09:40:53,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741891_1069 (size=4592) 2024-11-24T09:40:53,086 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/.tmp/cf2/42f951aaf5b34723a57e24a572fcce0f as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/42f951aaf5b34723a57e24a572fcce0f 2024-11-24T09:40:53,103 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in a47f41b92e82354ab3fd0381bbcca2bb/cf2 of a47f41b92e82354ab3fd0381bbcca2bb into 42f951aaf5b34723a57e24a572fcce0f(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T09:40:53,103 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for a47f41b92e82354ab3fd0381bbcca2bb: 2024-11-24T09:40:53,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=a47f41b92e82354ab3fd0381bbcca2bb, source=ba35ada4783f,36811,1732441233150, destination=ba35ada4783f,42723,1732441233325, warming up region on ba35ada4783f,42723,1732441233325 2024-11-24T09:40:53,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=a47f41b92e82354ab3fd0381bbcca2bb, source=ba35ada4783f,36811,1732441233150, destination=ba35ada4783f,42723,1732441233325, running balancer 2024-11-24T09:40:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE 2024-11-24T09:40:53,115 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE 2024-11-24T09:40:53,118 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=CLOSING, regionLocation=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:53,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(7855): Warmup {ENCODED => a47f41b92e82354ab3fd0381bbcca2bb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:53,121 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,122 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf1 2024-11-24T09:40:53,122 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:53,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE because future has completed 2024-11-24T09:40:53,125 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T09:40:53,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150}] 2024-11-24T09:40:53,136 INFO [StoreFileOpener-a47f41b92e82354ab3fd0381bbcca2bb-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2712e6288b4c462a98927f61046a782a 2024-11-24T09:40:53,137 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/2712e6288b4c462a98927f61046a782a 2024-11-24T09:40:53,151 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/e69dcfaf066c4daf9f1195642ec0ee41 2024-11-24T09:40:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T09:40:53,171 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/ff9108722adf4af1be1c05334e2a9394 2024-11-24T09:40:53,172 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:53,172 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,174 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf2 2024-11-24T09:40:53,174 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:53,192 INFO [StoreFileOpener-a47f41b92e82354ab3fd0381bbcca2bb-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e45189b6e4049218c48a4367d2c5670 2024-11-24T09:40:53,192 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/1e45189b6e4049218c48a4367d2c5670 2024-11-24T09:40:53,201 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/42f951aaf5b34723a57e24a572fcce0f 2024-11-24T09:40:53,201 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(1722): Closing a47f41b92e82354ab3fd0381bbcca2bb, disabling compactions & flushes 2024-11-24T09:40:53,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. after waiting 0 ms 2024-11-24T09:40:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42723 {}] regionserver.HRegion(1676): Region close journal for a47f41b92e82354ab3fd0381bbcca2bb: Waiting for close lock at 1732441253202Disabling compacts and flushes for region at 1732441253202Disabling writes for close at 1732441253202Writing region close event to WAL at 1732441253214 (+12 ms)Closed at 1732441253214 2024-11-24T09:40:53,280 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,280 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T09:40:53,280 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing a47f41b92e82354ab3fd0381bbcca2bb, disabling compactions & flushes 2024-11-24T09:40:53,281 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,281 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,281 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. after waiting 0 ms 2024-11-24T09:40:53,281 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,281 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/e69dcfaf066c4daf9f1195642ec0ee41, hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/2712e6288b4c462a98927f61046a782a] to archive 2024-11-24T09:40:53,284 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T09:40:53,293 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/e69dcfaf066c4daf9f1195642ec0ee41 to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/e69dcfaf066c4daf9f1195642ec0ee41 2024-11-24T09:40:53,299 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/2712e6288b4c462a98927f61046a782a to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/2712e6288b4c462a98927f61046a782a 2024-11-24T09:40:53,339 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/1e45189b6e4049218c48a4367d2c5670] to archive 2024-11-24T09:40:53,340 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T09:40:53,344 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/1e45189b6e4049218c48a4367d2c5670 to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/1e45189b6e4049218c48a4367d2c5670 2024-11-24T09:40:53,351 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T09:40:53,352 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,352 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for a47f41b92e82354ab3fd0381bbcca2bb: Waiting for close lock at 1732441253280Running coprocessor pre-close hooks at 1732441253280Disabling compacts and flushes for region at 1732441253280Disabling writes for close at 1732441253281 (+1 ms)Writing region close event to WAL at 1732441253347 (+66 ms)Running coprocessor post-close hooks at 1732441253352 (+5 ms)Closed at 1732441253352 2024-11-24T09:40:53,352 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding a47f41b92e82354ab3fd0381bbcca2bb move to ba35ada4783f,42723,1732441233325 record at close sequenceid=12 2024-11-24T09:40:53,357 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,359 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=CLOSED 2024-11-24T09:40:53,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150 because future has completed 2024-11-24T09:40:53,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T09:40:53,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150 in 239 msec 2024-11-24T09:40:53,370 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE; state=CLOSED, location=ba35ada4783f,42723,1732441233325; forceNewPlan=false, retain=false 2024-11-24T09:40:53,521 INFO [ba35ada4783f:43385 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T09:40:53,521 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=OPENING, regionLocation=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:53,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE because future has completed 2024-11-24T09:40:53,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325}] 2024-11-24T09:40:53,687 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,687 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => a47f41b92e82354ab3fd0381bbcca2bb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:53,687 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,688 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:53,688 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,688 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,693 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,707 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf1 2024-11-24T09:40:53,707 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:53,720 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/ff9108722adf4af1be1c05334e2a9394 2024-11-24T09:40:53,720 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:53,720 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,726 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf2 2024-11-24T09:40:53,726 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:53,741 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/42f951aaf5b34723a57e24a572fcce0f 2024-11-24T09:40:53,741 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:53,741 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,742 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,744 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,745 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,745 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,746 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T09:40:53,749 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,750 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened a47f41b92e82354ab3fd0381bbcca2bb; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68877844, jitterRate=0.026359856128692627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T09:40:53,750 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,751 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for a47f41b92e82354ab3fd0381bbcca2bb: Running coprocessor pre-open hook at 1732441253688Writing region info on filesystem at 1732441253688Initializing all the Stores at 1732441253689 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441253689Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441253693 (+4 ms)Cleaning up temporary data from old regions at 1732441253745 (+52 ms)Running coprocessor post-open hooks at 1732441253751 (+6 ms)Region opened successfully at 1732441253751 2024-11-24T09:40:53,753 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., pid=12, masterSystemTime=1732441253680 2024-11-24T09:40:53,757 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,757 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,758 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=OPEN, openSeqNum=18, regionLocation=ba35ada4783f,42723,1732441233325 2024-11-24T09:40:53,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325 because future has completed 2024-11-24T09:40:53,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T09:40:53,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,42723,1732441233325 in 239 msec 2024-11-24T09:40:53,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, REOPEN/MOVE in 658 msec 2024-11-24T09:40:53,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:40:53,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:40:53,833 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ba35ada4783f,42723,1732441233325: testing ***** 2024-11-24T09:40:53,833 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T09:40:53,835 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T09:40:53,837 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T09:40:53,860 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T09:40:53,862 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T09:40:53,880 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 495526400 }, "NonHeapMemoryUsage": { "committed": 171311104, "init": 7667712, "max": -1, "used": 168806176 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ba35ada4783f", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10264, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 100, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 36, "ProcessCallTime_98th_percentile": 100, "ProcessCallTime_99th_percentile": 100, "ProcessCallTime_99.9th_percentile": 100, "ProcessCallTime_TimeRangeCount_0-1": 7, "ProcessCallTime_TimeRangeCount_1-3": 17, "ProcessCallTime_TimeRangeCount_3-10": 4, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 2, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 100, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 36, "TotalCallTime_98th_percentile": 100, "TotalCallTime_99th_percentile": 100, "TotalCallTime_99.9th_percentile": 100, "TotalCallTime_TimeRangeCount_0-1": 6, "TotalCallTime_TimeRangeCount_1-3": 18, "TotalCallTime_TimeRangeCount_3-10": 4, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 2, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ba35ada4783f", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52387", "tag.serverName": "ba35ada4783f,42723,1732441233325", "tag.clusterId": "f30c0795-04ef-4a63-a29c-8fba537cfc7d", "tag.Context": "regionserver", "tag.Hostname": "ba35ada4783f", "regionCount": 2, "storeCount": 4, "hlogFileCount": 6, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732441233325, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 599040, "ByteBuffAllocatorHeapAllocationRatio": 0.35775729671400613, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 29, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 17, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "ScanTime_TimeRangeCount_1-3": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 16, "Put_min": 2, "Put_max": 3, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 14, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 103, "FlushTime_max": 103, "FlushTime_mean": 103, "FlushTime_25th_percentile": 103, "FlushTime_median": 103, "FlushTime_75th_percentile": 103, "FlushTime_90th_percentile": 103, "FlushTime_95th_percentile": 103, "FlushTime_98th_percentile": 103, "FlushTime_99th_percentile": 103, "FlushTime_99.9th_percentile": 103, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 64, "ScanSize_max": 1112, "ScanSize_mean": 902, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_10-100": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T09:40:53,883 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43385 {}] master.MasterRpcServices(700): ba35ada4783f,42723,1732441233325 reported a fatal error: ***** ABORTING region server ba35ada4783f,42723,1732441233325: testing ***** 2024-11-24T09:40:53,888 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba35ada4783f,42723,1732441233325' ***** 2024-11-24T09:40:53,888 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T09:40:53,888 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T09:40:53,888 INFO [RS:1;ba35ada4783f:42723 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T09:40:53,889 INFO [RS:1;ba35ada4783f:42723 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T09:40:53,889 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(3091): Received CLOSE for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,889 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(956): aborting server ba35ada4783f,42723,1732441233325 2024-11-24T09:40:53,889 INFO [RS:1;ba35ada4783f:42723 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:40:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36811 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:55662 deadline: 1732441313889, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=42723 startCode=1732441233325. As of locationSeqNum=12. 2024-11-24T09:40:53,889 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a47f41b92e82354ab3fd0381bbcca2bb, disabling compactions & flushes 2024-11-24T09:40:53,890 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T09:40:53,890 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,890 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,890 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. after waiting 0 ms 2024-11-24T09:40:53,890 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,890 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,36811,1732441233150, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,36811,1732441233150, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=42723 startCode=1732441233325. As of locationSeqNum=12. 2024-11-24T09:40:53,890 INFO [RS:1;ba35ada4783f:42723 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ba35ada4783f:42723. 2024-11-24T09:40:53,890 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,36811,1732441233150, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=42723 startCode=1732441233325. As of locationSeqNum=12. 2024-11-24T09:40:53,890 DEBUG [RS:1;ba35ada4783f:42723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:40:53,890 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,36811,1732441233150, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ba35ada4783f port=42723 startCode=1732441233325. As of locationSeqNum=12. 2024-11-24T09:40:53,891 DEBUG [RS:1;ba35ada4783f:42723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:40:53,891 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T09:40:53,891 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T09:40:53,891 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T09:40:53,891 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T09:40:53,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,42723,1732441233325 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:53,897 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,42723,1732441233325 aborting 2024-11-24T09:40:53,897 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,42723,1732441233325 aborting 2024-11-24T09:40:53,897 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 from cache 2024-11-24T09:40:53,900 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T09:40:53,900 DEBUG [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, a47f41b92e82354ab3fd0381bbcca2bb=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.} 2024-11-24T09:40:53,901 DEBUG [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:53,901 INFO [regionserver/ba35ada4783f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:40:53,901 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T09:40:53,901 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T09:40:53,901 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T09:40:53,901 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T09:40:53,901 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T09:40:53,902 ERROR [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-24T09:40:53,902 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T09:40:53,903 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T09:40:53,903 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732441253901Running coprocessor pre-close hooks at 1732441253901Disabling compacts and flushes for region at 1732441253901Disabling writes for close at 1732441253901Writing region close event to WAL at 1732441253902 (+1 ms)Running coprocessor post-close hooks at 1732441253902Closed at 1732441253903 (+1 ms) 2024-11-24T09:40:53,903 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T09:40:53,903 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:53,904 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a47f41b92e82354ab3fd0381bbcca2bb: Waiting for close lock at 1732441253889Running coprocessor pre-close hooks at 1732441253889Disabling compacts and flushes for region at 1732441253889Disabling writes for close at 1732441253890 (+1 ms)Writing region close event to WAL at 1732441253903 (+13 ms)Running coprocessor post-close hooks at 1732441253903Closed at 1732441253903 2024-11-24T09:40:53,904 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:54,002 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:40:54,005 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1] 2024-11-24T09:40:54,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,42723,1732441233325 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:54,007 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,42723,1732441233325 aborting 2024-11-24T09:40:54,008 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ba35ada4783f,42723,1732441233325 aborting 2024-11-24T09:40:54,008 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 from cache 2024-11-24T09:40:54,101 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(976): stopping server ba35ada4783f,42723,1732441233325; all regions closed. 2024-11-24T09:40:54,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741836_1012 (size=2747) 2024-11-24T09:40:54,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741836_1012 (size=2747) 2024-11-24T09:40:54,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741836_1012 (size=2747) 2024-11-24T09:40:54,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741834_1010 (size=1677) 2024-11-24T09:40:54,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741834_1010 (size=1677) 2024-11-24T09:40:54,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741834_1010 (size=1677) 2024-11-24T09:40:54,117 DEBUG [RS:1;ba35ada4783f:42723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:40:54,117 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:40:54,117 INFO [RS:1;ba35ada4783f:42723 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:40:54,117 INFO [RS:1;ba35ada4783f:42723 {}] hbase.ChoreService(370): Chore service for: regionserver/ba35ada4783f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T09:40:54,117 INFO [RS:1;ba35ada4783f:42723 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:40:54,118 INFO [regionserver/ba35ada4783f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:40:54,118 INFO [RS:1;ba35ada4783f:42723 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42723 2024-11-24T09:40:54,189 INFO [RS:1;ba35ada4783f:42723 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:40:54,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba35ada4783f,42723,1732441233325 2024-11-24T09:40:54,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:40:54,189 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$357/0x00007f35ac8e0aa0@703cf93 rejected from java.util.concurrent.ThreadPoolExecutor@42014117[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 12] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-24T09:40:54,191 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba35ada4783f,42723,1732441233325] 2024-11-24T09:40:54,205 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba35ada4783f,42723,1732441233325 already deleted, retry=false 2024-11-24T09:40:54,206 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ba35ada4783f,42723,1732441233325 on ba35ada4783f,43385,1732441231965 2024-11-24T09:40:54,211 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ba35ada4783f,42723,1732441233325, splitWal=true, meta=true 2024-11-24T09:40:54,213 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:40:54,214 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1] 2024-11-24T09:40:54,216 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ba35ada4783f,42723,1732441233325, splitWal=true, meta=true 2024-11-24T09:40:54,216 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ba35ada4783f:42723 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ba35ada4783f/172.17.0.2:42723 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:54,218 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ba35ada4783f,42723,1732441233325, splitWal=true, meta=true, isMeta: true 2024-11-24T09:40:54,218 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1, error=java.net.ConnectException: Call to address=ba35ada4783f:42723 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ba35ada4783f/172.17.0.2:42723 2024-11-24T09:40:54,218 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-24T09:40:54,218 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 from cache 2024-11-24T09:40:54,219 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address ba35ada4783f:42723 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ba35ada4783f/172.17.0.2:42723 2024-11-24T09:40:54,221 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for ba35ada4783f,42723,1732441233325 (carryingMeta=true) ba35ada4783f,42723,1732441233325/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2792ee9e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T09:40:54,221 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting 2024-11-24T09:40:54,222 INFO [PEWorker-1 {}] master.SplitWALManager(105): ba35ada4783f,42723,1732441233325 WAL count=1, meta=true 2024-11-24T09:40:54,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta}] 2024-11-24T09:40:54,231 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=ba35ada4783f,35917,1732441233385 2024-11-24T09:40:54,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, worker=ba35ada4783f,35917,1732441233385}] 2024-11-24T09:40:54,298 INFO [RS:1;ba35ada4783f:42723 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:40:54,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:40:54,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x1016d80e1650002, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:40:54,298 INFO [RS:1;ba35ada4783f:42723 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba35ada4783f,42723,1732441233325; zookeeper connection closed. 2024-11-24T09:40:54,298 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@aa9445a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@aa9445a 2024-11-24T09:40:54,399 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:40:54,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44971, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:40:54,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T09:40:54,419 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, size=2.7 K (2747bytes) 2024-11-24T09:40:54,419 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta 2024-11-24T09:40:54,419 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta after 0ms 2024-11-24T09:40:54,422 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:54,422 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta took 3ms 2024-11-24T09:40:54,428 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-24T09:40:54,429 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta so closing down 2024-11-24T09:40:54,429 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:54,431 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta.temp 2024-11-24T09:40:54,434 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta.temp 2024-11-24T09:40:54,435 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:54,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741892_1070 (size=2527) 2024-11-24T09:40:54,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741892_1070 (size=2527) 2024-11-24T09:40:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741892_1070 (size=2527) 2024-11-24T09:40:54,460 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T09:40:54,462 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta.temp to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T09:40:54,462 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 40 ms; skipped=1; WAL=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, size=2.7 K, length=2747, corrupted=false, cancelled=false 2024-11-24T09:40:54,462 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, journal: Splitting hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, size=2.7 K (2747bytes) at 1732441254419Finishing writing output for hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta so closing down at 1732441254429 (+10 ms)Creating recovered edits writer path=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta.temp at 1732441254434 (+5 ms)3 split writer threads finished at 1732441254435 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732441254460 (+25 ms)Rename recovered edits hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta.temp to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732441254462 (+2 ms)Processed 16 edits across 1 Regions in 40 ms; skipped=1; WAL=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, size=2.7 K, length=2747, corrupted=false, cancelled=false at 1732441254462 2024-11-24T09:40:54,462 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta 2024-11-24T09:40:54,464 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T09:40:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T09:40:54,473 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs 2024-11-24T09:40:54,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T09:40:54,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, worker=ba35ada4783f,35917,1732441233385 in 239 msec 2024-11-24T09:40:54,478 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=ba35ada4783f,35917,1732441233385 2024-11-24T09:40:54,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T09:40:54,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure ba35ada4783f%2C42723%2C1732441233325.meta.1732441235913.meta, worker=ba35ada4783f,35917,1732441233385 in 255 msec 2024-11-24T09:40:54,485 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting dir is empty, no logs to split. 2024-11-24T09:40:54,485 INFO [PEWorker-1 {}] master.SplitWALManager(105): ba35ada4783f,42723,1732441233325 WAL count=0, meta=true 2024-11-24T09:40:54,485 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ba35ada4783f,42723,1732441233325 WAL splitting is done? wals=0, meta=true 2024-11-24T09:40:54,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T09:40:54,489 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:40:54,490 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T09:40:54,523 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:40:54,524 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1] 2024-11-24T09:40:54,525 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to ba35ada4783f:42723 this server is in the failed servers list 2024-11-24T09:40:54,525 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ba35ada4783f:42723 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:42723 2024-11-24T09:40:54,525 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:42723 2024-11-24T09:40:54,525 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 from cache 2024-11-24T09:40:54,640 DEBUG [ba35ada4783f:43385 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T09:40:54,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(204): Hosts are {ba35ada4783f=0} racks are {/default-rack=0} 2024-11-24T09:40:54,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T09:40:54,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T09:40:54,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T09:40:54,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T09:40:54,641 INFO [ba35ada4783f:43385 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T09:40:54,641 INFO [ba35ada4783f:43385 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T09:40:54,641 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T09:40:54,642 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:54,644 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,36811,1732441233150, state=OPENING 2024-11-24T09:40:54,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:54,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:54,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:54,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:54,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:54,673 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:40:54,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:54,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,36811,1732441233150}] 2024-11-24T09:40:54,831 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T09:40:54,831 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:54,831 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T09:40:54,833 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C36811%2C1732441233150.meta, suffix=.meta, logDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,36811,1732441233150, archiveDir=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs, maxLogs=32 2024-11-24T09:40:54,851 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,36811,1732441233150/ba35ada4783f%2C36811%2C1732441233150.meta.1732441254833.meta, exclude list is [], retry=0 2024-11-24T09:40:54,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:54,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:54,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:54,863 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,36811,1732441233150/ba35ada4783f%2C36811%2C1732441233150.meta.1732441254833.meta 2024-11-24T09:40:54,863 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:54,863 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:54,864 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T09:40:54,864 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T09:40:54,864 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T09:40:54,864 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T09:40:54,864 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:54,864 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T09:40:54,864 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T09:40:54,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T09:40:54,867 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T09:40:54,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:54,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:54,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T09:40:54,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T09:40:54,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:54,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:54,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T09:40:54,870 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T09:40:54,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:54,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:54,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T09:40:54,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T09:40:54,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:54,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:40:54,872 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T09:40:54,873 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740 2024-11-24T09:40:54,874 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740 2024-11-24T09:40:54,875 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T09:40:54,877 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:54,880 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T09:40:54,880 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-24T09:40:54,903 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/info/e191d42eec2449488c6a4dd470d9e2a5 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb./info:regioninfo/1732441253757/Put/seqid=0 2024-11-24T09:40:54,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741894_1072 (size=11177) 2024-11-24T09:40:54,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741894_1072 (size=11177) 2024-11-24T09:40:54,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741894_1072 (size=11177) 2024-11-24T09:40:54,915 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/info/e191d42eec2449488c6a4dd470d9e2a5 2024-11-24T09:40:54,947 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/ns/28d38b282d93409fb168e5236762c82a is 43, key is default/ns:d/1732441236108/Put/seqid=0 2024-11-24T09:40:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741895_1073 (size=5153) 2024-11-24T09:40:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741895_1073 (size=5153) 2024-11-24T09:40:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741895_1073 (size=5153) 2024-11-24T09:40:54,970 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/ns/28d38b282d93409fb168e5236762c82a 2024-11-24T09:40:54,992 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/table/ec023ec3cb9d4ad29af4a137659dfefb is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732441251719/Put/seqid=0 2024-11-24T09:40:54,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741896_1074 (size=5431) 2024-11-24T09:40:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741896_1074 (size=5431) 2024-11-24T09:40:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741896_1074 (size=5431) 2024-11-24T09:40:55,003 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/table/ec023ec3cb9d4ad29af4a137659dfefb 2024-11-24T09:40:55,018 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/info/e191d42eec2449488c6a4dd470d9e2a5 as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/info/e191d42eec2449488c6a4dd470d9e2a5 2024-11-24T09:40:55,026 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/info/e191d42eec2449488c6a4dd470d9e2a5, entries=36, sequenceid=18, filesize=10.9 K 2024-11-24T09:40:55,027 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/ns/28d38b282d93409fb168e5236762c82a as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/ns/28d38b282d93409fb168e5236762c82a 2024-11-24T09:40:55,032 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:40:55,034 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,36811,1732441233150, seqNum=-1] 2024-11-24T09:40:55,034 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/ns/28d38b282d93409fb168e5236762c82a, entries=2, sequenceid=18, filesize=5.0 K 2024-11-24T09:40:55,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36811 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:55662 deadline: 1732441315034, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on ba35ada4783f,36811,1732441233150 2024-11-24T09:40:55,035 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,36811,1732441233150, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,36811,1732441233150, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on ba35ada4783f,36811,1732441233150 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T09:40:55,035 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,36811,1732441233150, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on ba35ada4783f,36811,1732441233150 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T09:40:55,036 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/table/ec023ec3cb9d4ad29af4a137659dfefb as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/table/ec023ec3cb9d4ad29af4a137659dfefb 2024-11-24T09:40:55,036 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,36811,1732441233150, seqNum=-1 from cache 2024-11-24T09:40:55,044 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/table/ec023ec3cb9d4ad29af4a137659dfefb, entries=2, sequenceid=18, filesize=5.3 K 2024-11-24T09:40:55,044 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 164ms, sequenceid=18, compaction requested=false; wal=null 2024-11-24T09:40:55,045 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T09:40:55,047 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T09:40:55,047 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T09:40:55,048 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:40:55,049 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T09:40:55,053 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T09:40:55,054 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63266316, jitterRate=-0.057258427143096924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:40:55,054 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T09:40:55,055 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732441254865Writing region info on filesystem at 1732441254865Initializing all the Stores at 1732441254866 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441254866Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441254866Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441254866Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732441254866Obtaining lock to block concurrent updates at 1732441254880 (+14 ms)Preparing flush snapshotting stores in 1588230740 at 1732441254880Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732441254880Flushing stores of hbase:meta,,1.1588230740 at 1732441254880Flushing 1588230740/info: creating writer at 1732441254881 (+1 ms)Flushing 1588230740/info: appending metadata at 1732441254903 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732441254903Flushing 1588230740/ns: creating writer at 1732441254925 (+22 ms)Flushing 1588230740/ns: appending metadata at 1732441254946 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732441254946Flushing 1588230740/table: creating writer at 1732441254976 (+30 ms)Flushing 1588230740/table: appending metadata at 1732441254991 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732441254991Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7233a026: reopening flushed file at 1732441255017 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@914e625: reopening flushed file at 1732441255026 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b17905d: reopening flushed file at 1732441255034 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 164ms, sequenceid=18, compaction requested=false; wal=null at 1732441255044 (+10 ms)Cleaning up temporary data from old regions at 1732441255047 (+3 ms)Running coprocessor post-open hooks at 1732441255054 (+7 ms)Region opened successfully at 1732441255055 (+1 ms) 2024-11-24T09:40:55,057 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732441254827 2024-11-24T09:40:55,060 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T09:40:55,060 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T09:40:55,061 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:55,062 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,36811,1732441233150, state=OPEN 2024-11-24T09:40:55,070 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43385 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba35ada4783f,36811,1732441233150, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T09:40:55,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:55,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:55,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:40:55,105 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:55,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:55,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:55,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:40:55,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T09:40:55,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,36811,1732441233150 in 431 msec 2024-11-24T09:40:55,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T09:40:55,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 621 msec 2024-11-24T09:40:55,112 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(207): ba35ada4783f,42723,1732441233325 had 2 regions 2024-11-24T09:40:55,113 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ba35ada4783f,42723,1732441233325, splitWal=true, meta=true, isMeta: false 2024-11-24T09:40:55,115 INFO [PEWorker-4 {}] master.SplitWALManager(105): ba35ada4783f,42723,1732441233325 WAL count=1, meta=false 2024-11-24T09:40:55,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ba35ada4783f%2C42723%2C1732441233325.1732441235469}] 2024-11-24T09:40:55,118 DEBUG [PEWorker-5 {}] master.SplitWALManager(158): Acquired split WAL worker=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:55,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ba35ada4783f%2C42723%2C1732441233325.1732441235469, worker=ba35ada4783f,36811,1732441233150}] 2024-11-24T09:40:55,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36811 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-24T09:40:55,289 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469, size=1.6 K (1677bytes) 2024-11-24T09:40:55,289 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469 2024-11-24T09:40:55,290 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469 after 1ms 2024-11-24T09:40:55,292 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:55,293 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469 took 4ms 2024-11-24T09:40:55,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for a47f41b92e82354ab3fd0381bbcca2bb: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T09:40:55,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469 so closing down 2024-11-24T09:40:55,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:55,314 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:55,315 INFO [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 16 ms; skipped=6; WAL=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-11-24T09:40:55,315 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469, journal: Splitting hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469, size=1.6 K (1677bytes) at 1732441255289Finishing writing output for hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469 so closing down at 1732441255302 (+13 ms)3 split writer threads finished at 1732441255314 (+12 ms)Processed 6 edits across 0 Regions in 16 ms; skipped=6; WAL=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1732441255315 (+1 ms) 2024-11-24T09:40:55,315 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469 2024-11-24T09:40:55,315 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-24T09:40:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43385 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-24T09:40:55,321 INFO [PEWorker-2 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting/ba35ada4783f%2C42723%2C1732441233325.1732441235469 to hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs 2024-11-24T09:40:55,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-24T09:40:55,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba35ada4783f%2C42723%2C1732441233325.1732441235469, worker=ba35ada4783f,36811,1732441233150 in 202 msec 2024-11-24T09:40:55,326 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:55,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-24T09:40:55,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure ba35ada4783f%2C42723%2C1732441233325.1732441235469, worker=ba35ada4783f,36811,1732441233150 in 211 msec 2024-11-24T09:40:55,329 INFO [PEWorker-4 {}] master.SplitLogManager(171): hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/WALs/ba35ada4783f,42723,1732441233325-splitting dir is empty, no logs to split. 2024-11-24T09:40:55,329 INFO [PEWorker-4 {}] master.SplitWALManager(105): ba35ada4783f,42723,1732441233325 WAL count=0, meta=false 2024-11-24T09:40:55,329 DEBUG [PEWorker-4 {}] procedure.ServerCrashProcedure(329): Check if ba35ada4783f,42723,1732441233325 WAL splitting is done? wals=0, meta=false 2024-11-24T09:40:55,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN}] 2024-11-24T09:40:55,335 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN 2024-11-24T09:40:55,336 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T09:40:55,487 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(204): Hosts are {ba35ada4783f=0} racks are {/default-rack=0} 2024-11-24T09:40:55,487 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T09:40:55,487 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T09:40:55,487 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T09:40:55,487 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T09:40:55,487 INFO [ba35ada4783f:43385 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T09:40:55,487 INFO [ba35ada4783f:43385 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T09:40:55,487 DEBUG [ba35ada4783f:43385 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T09:40:55,487 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=OPENING, regionLocation=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:55,488 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ba35ada4783f:42723 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba35ada4783f/172.17.0.2:42723 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:55,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1, error=java.net.ConnectException: Call to address=ba35ada4783f:42723 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba35ada4783f/172.17.0.2:42723 2024-11-24T09:40:55,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T09:40:55,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42723,1732441233325, seqNum=-1 from cache 2024-11-24T09:40:55,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ba35ada4783f:42723 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba35ada4783f/172.17.0.2:42723 2024-11-24T09:40:55,592 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:40:55,592 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,36811,1732441233150, seqNum=-1] 2024-11-24T09:40:55,593 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:40:55,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:40:55,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN because future has completed 2024-11-24T09:40:55,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150}] 2024-11-24T09:40:55,770 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:55,770 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => a47f41b92e82354ab3fd0381bbcca2bb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:55,770 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,770 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:55,770 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,770 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,772 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,774 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf1 2024-11-24T09:40:55,774 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:55,788 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf1/ff9108722adf4af1be1c05334e2a9394 2024-11-24T09:40:55,788 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:55,788 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,789 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a47f41b92e82354ab3fd0381bbcca2bb columnFamilyName cf2 2024-11-24T09:40:55,789 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:55,798 DEBUG [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/cf2/42f951aaf5b34723a57e24a572fcce0f 2024-11-24T09:40:55,798 INFO [StoreOpener-a47f41b92e82354ab3fd0381bbcca2bb-1 {}] regionserver.HStore(327): Store=a47f41b92e82354ab3fd0381bbcca2bb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:55,799 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,800 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,801 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,802 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,802 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,802 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T09:40:55,805 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,806 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened a47f41b92e82354ab3fd0381bbcca2bb; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73135555, jitterRate=0.08980469405651093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T09:40:55,806 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:40:55,807 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for a47f41b92e82354ab3fd0381bbcca2bb: Running coprocessor pre-open hook at 1732441255771Writing region info on filesystem at 1732441255771Initializing all the Stores at 1732441255772 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441255772Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441255772Cleaning up temporary data from old regions at 1732441255802 (+30 ms)Running coprocessor post-open hooks at 1732441255806 (+4 ms)Region opened successfully at 1732441255806 2024-11-24T09:40:55,808 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., pid=21, masterSystemTime=1732441255766 2024-11-24T09:40:55,810 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:55,810 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:40:55,811 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=a47f41b92e82354ab3fd0381bbcca2bb, regionState=OPEN, openSeqNum=18, regionLocation=ba35ada4783f,36811,1732441233150 2024-11-24T09:40:55,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150 because future has completed 2024-11-24T09:40:55,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-24T09:40:55,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure a47f41b92e82354ab3fd0381bbcca2bb, server=ba35ada4783f,36811,1732441233150 in 219 msec 2024-11-24T09:40:55,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-24T09:40:55,827 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server ba35ada4783f,42723,1732441233325 after splitting done 2024-11-24T09:40:55,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=a47f41b92e82354ab3fd0381bbcca2bb, ASSIGN in 489 msec 2024-11-24T09:40:55,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure ba35ada4783f,42723,1732441233325, splitWal=true, meta=true in 1.6200 sec 2024-11-24T09:40:56,043 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:40:56,044 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,36811,1732441233150, seqNum=-1] 2024-11-24T09:40:56,172 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ba35ada4783f:42723 this server is in the failed servers list 2024-11-24T09:40:56,173 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ba35ada4783f:42723 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:42723 2024-11-24T09:40:56,173 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ba35ada4783f:42723 2024-11-24T09:40:56,173 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,42723,1732441233325, seqNum=12 from cache 2024-11-24T09:40:56,390 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb., hostname=ba35ada4783f,36811,1732441233150, seqNum=18] 2024-11-24T09:40:56,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=404 (was 411), OpenFileDescriptor=1040 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=590 (was 590), ProcessCount=11 (was 11), AvailableMemoryMB=7750 (was 7798) 2024-11-24T09:40:56,407 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-11-24T09:40:56,420 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=404, OpenFileDescriptor=1040, MaxFileDescriptor=1048576, SystemLoadAverage=590, ProcessCount=11, AvailableMemoryMB=7749 2024-11-24T09:40:56,420 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-11-24T09:40:56,437 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:56,439 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:56,440 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:56,442 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15702946, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-15702946, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:56,459 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15702946/hregion-15702946.1732441256443, exclude list is [], retry=0 2024-11-24T09:40:56,462 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:56,462 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:56,464 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:56,466 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15702946/hregion-15702946.1732441256443 2024-11-24T09:40:56,468 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:56,468 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4b29e232cae310ac5d117f55b573863f, NAME => 'testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:40:56,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741898_1076 (size=67) 2024-11-24T09:40:56,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741898_1076 (size=67) 2024-11-24T09:40:56,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741898_1076 (size=67) 2024-11-24T09:40:56,504 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:56,506 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,507 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName a 2024-11-24T09:40:56,508 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:56,508 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:56,508 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,509 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName b 2024-11-24T09:40:56,510 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:56,510 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:56,510 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,512 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName c 2024-11-24T09:40:56,512 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:56,512 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:56,512 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,513 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,513 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,514 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,514 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,515 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:56,516 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,518 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:56,519 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b29e232cae310ac5d117f55b573863f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66223993, jitterRate=-0.01318560540676117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:56,520 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b29e232cae310ac5d117f55b573863f: Writing region info on filesystem at 1732441256504Initializing all the Stores at 1732441256505 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441256505Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441256506 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441256506Cleaning up temporary data from old regions at 1732441256514 (+8 ms)Region opened successfully at 1732441256520 (+6 ms) 2024-11-24T09:40:56,520 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b29e232cae310ac5d117f55b573863f, disabling compactions & flushes 2024-11-24T09:40:56,520 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. 2024-11-24T09:40:56,520 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. 2024-11-24T09:40:56,520 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. after waiting 0 ms 2024-11-24T09:40:56,520 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. 2024-11-24T09:40:56,521 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. 2024-11-24T09:40:56,521 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b29e232cae310ac5d117f55b573863f: Waiting for close lock at 1732441256520Disabling compacts and flushes for region at 1732441256520Disabling writes for close at 1732441256520Writing region close event to WAL at 1732441256520Closed at 1732441256520 2024-11-24T09:40:56,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741897_1075 (size=93) 2024-11-24T09:40:56,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741897_1075 (size=93) 2024-11-24T09:40:56,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741897_1075 (size=93) 2024-11-24T09:40:56,530 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:56,530 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15702946:(num 1732441256443) 2024-11-24T09:40:56,530 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:56,532 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:56,548 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533, exclude list is [], retry=0 2024-11-24T09:40:56,550 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:56,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:56,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:56,554 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533 2024-11-24T09:40:56,554 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:40:56,554 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b29e232cae310ac5d117f55b573863f, NAME => 'testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:56,554 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:56,554 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,555 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,556 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,557 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName a 2024-11-24T09:40:56,557 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:56,557 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:56,558 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,558 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName b 2024-11-24T09:40:56,559 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:56,559 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:56,559 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,560 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName c 2024-11-24T09:40:56,560 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:56,560 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:56,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,561 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,564 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,566 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,566 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,567 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:56,568 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:56,569 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b29e232cae310ac5d117f55b573863f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64335351, jitterRate=-0.041328564286231995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:56,570 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b29e232cae310ac5d117f55b573863f: Writing region info on filesystem at 1732441256555Initializing all the Stores at 1732441256556 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441256556Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441256556Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441256556Cleaning up temporary data from old regions at 1732441256566 (+10 ms)Region opened successfully at 1732441256570 (+4 ms) 2024-11-24T09:40:56,618 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b29e232cae310ac5d117f55b573863f 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T09:40:56,645 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/a/28a2554a646b44b2bed756c2cbc34b7c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732441256570/Put/seqid=0 2024-11-24T09:40:56,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741900_1078 (size=5958) 2024-11-24T09:40:56,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741900_1078 (size=5958) 2024-11-24T09:40:56,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741900_1078 (size=5958) 2024-11-24T09:40:56,681 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/a/28a2554a646b44b2bed756c2cbc34b7c 2024-11-24T09:40:56,712 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/b/06a74ed8639a46d7a4031300c7165715 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732441256583/Put/seqid=0 2024-11-24T09:40:56,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741901_1079 (size=5958) 2024-11-24T09:40:56,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741901_1079 (size=5958) 2024-11-24T09:40:56,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741901_1079 (size=5958) 2024-11-24T09:40:56,750 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/b/06a74ed8639a46d7a4031300c7165715 2024-11-24T09:40:56,786 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/c/f7b4855b51954b978a673203854f25be is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732441256607/Put/seqid=0 2024-11-24T09:40:56,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741902_1080 (size=5958) 2024-11-24T09:40:56,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741902_1080 (size=5958) 2024-11-24T09:40:56,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741902_1080 (size=5958) 2024-11-24T09:40:56,824 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/c/f7b4855b51954b978a673203854f25be 2024-11-24T09:40:56,833 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/a/28a2554a646b44b2bed756c2cbc34b7c as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/a/28a2554a646b44b2bed756c2cbc34b7c 2024-11-24T09:40:56,844 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/a/28a2554a646b44b2bed756c2cbc34b7c, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T09:40:56,845 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/b/06a74ed8639a46d7a4031300c7165715 as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/b/06a74ed8639a46d7a4031300c7165715 2024-11-24T09:40:56,853 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/b/06a74ed8639a46d7a4031300c7165715, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T09:40:56,854 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/c/f7b4855b51954b978a673203854f25be as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/c/f7b4855b51954b978a673203854f25be 2024-11-24T09:40:56,861 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/c/f7b4855b51954b978a673203854f25be, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T09:40:56,862 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4b29e232cae310ac5d117f55b573863f in 244ms, sequenceid=33, compaction requested=false 2024-11-24T09:40:56,862 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4b29e232cae310ac5d117f55b573863f: 2024-11-24T09:40:56,862 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b29e232cae310ac5d117f55b573863f, disabling compactions & flushes 2024-11-24T09:40:56,862 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. 2024-11-24T09:40:56,862 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. 2024-11-24T09:40:56,862 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. after waiting 0 ms 2024-11-24T09:40:56,862 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. 2024-11-24T09:40:56,865 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. 2024-11-24T09:40:56,865 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b29e232cae310ac5d117f55b573863f: Waiting for close lock at 1732441256862Disabling compacts and flushes for region at 1732441256862Disabling writes for close at 1732441256862Writing region close event to WAL at 1732441256865 (+3 ms)Closed at 1732441256865 2024-11-24T09:40:56,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741899_1077 (size=2873) 2024-11-24T09:40:56,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741899_1077 (size=2873) 2024-11-24T09:40:56,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741899_1077 (size=2873) 2024-11-24T09:40:56,876 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/b/06a74ed8639a46d7a4031300c7165715 to hdfs://localhost:43013/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/b/06a74ed8639a46d7a4031300c7165715 2024-11-24T09:40:56,897 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533, size=2.8 K (2873bytes) 2024-11-24T09:40:56,897 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533 2024-11-24T09:40:56,897 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533 after 0ms 2024-11-24T09:40:56,908 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:56,908 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533 took 11ms 2024-11-24T09:40:56,928 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533 so closing down 2024-11-24T09:40:56,928 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:40:56,932 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441256533.temp 2024-11-24T09:40:56,946 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000003-wal.1732441256533.temp 2024-11-24T09:40:56,946 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:40:57,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741903_1081 (size=2312) 2024-11-24T09:40:57,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741903_1081 (size=2312) 2024-11-24T09:40:57,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741903_1081 (size=2312) 2024-11-24T09:40:57,035 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000003-wal.1732441256533.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T09:40:57,037 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000003-wal.1732441256533.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000032 2024-11-24T09:40:57,038 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 113 ms; skipped=2; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-11-24T09:40:57,038 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533, journal: Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533, size=2.8 K (2873bytes) at 1732441256897Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533 so closing down at 1732441256928 (+31 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000003-wal.1732441256533.temp at 1732441256946 (+18 ms)3 split writer threads finished at 1732441256946Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000003-wal.1732441256533.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732441257036 (+90 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000003-wal.1732441256533.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000032 at 1732441257037 (+1 ms)Processed 32 edits across 1 Regions in 113 ms; skipped=2; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1732441257038 (+1 ms) 2024-11-24T09:40:57,041 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441256533 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441256533 2024-11-24T09:40:57,042 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000032 2024-11-24T09:40:57,042 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:57,045 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:57,065 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441257045, exclude list is [], retry=0 2024-11-24T09:40:57,072 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:57,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:57,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:57,091 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441257045 2024-11-24T09:40:57,091 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:57,092 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b29e232cae310ac5d117f55b573863f, NAME => 'testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:57,096 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:57,096 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,096 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,100 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,101 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName a 2024-11-24T09:40:57,101 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,125 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/a/28a2554a646b44b2bed756c2cbc34b7c 2024-11-24T09:40:57,125 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,126 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,131 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName b 2024-11-24T09:40:57,131 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,132 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,132 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,134 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29e232cae310ac5d117f55b573863f columnFamilyName c 2024-11-24T09:40:57,134 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,156 DEBUG [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/c/f7b4855b51954b978a673203854f25be 2024-11-24T09:40:57,156 INFO [StoreOpener-4b29e232cae310ac5d117f55b573863f-1 {}] regionserver.HStore(327): Store=4b29e232cae310ac5d117f55b573863f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,156 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,157 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,160 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,160 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000032 2024-11-24T09:40:57,163 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:40:57,165 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000032 2024-11-24T09:40:57,165 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b29e232cae310ac5d117f55b573863f 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T09:40:57,184 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/b/d3766795fe0346f79e38835df9e08dcf is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732441256583/Put/seqid=0 2024-11-24T09:40:57,202 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T09:40:57,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741905_1083 (size=5958) 2024-11-24T09:40:57,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741905_1083 (size=5958) 2024-11-24T09:40:57,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741905_1083 (size=5958) 2024-11-24T09:40:57,242 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/b/d3766795fe0346f79e38835df9e08dcf 2024-11-24T09:40:57,252 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/.tmp/b/d3766795fe0346f79e38835df9e08dcf as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/b/d3766795fe0346f79e38835df9e08dcf 2024-11-24T09:40:57,262 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/b/d3766795fe0346f79e38835df9e08dcf, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T09:40:57,262 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4b29e232cae310ac5d117f55b573863f in 97ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T09:40:57,263 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/0000000000000000032 2024-11-24T09:40:57,266 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,266 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,268 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:57,271 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b29e232cae310ac5d117f55b573863f 2024-11-24T09:40:57,284 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/4b29e232cae310ac5d117f55b573863f/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T09:40:57,286 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b29e232cae310ac5d117f55b573863f; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59544421, jitterRate=-0.11271898448467255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:57,287 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b29e232cae310ac5d117f55b573863f: Writing region info on filesystem at 1732441257096Initializing all the Stores at 1732441257098 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257098Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257100 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257100Obtaining lock to block concurrent updates at 1732441257165 (+65 ms)Preparing flush snapshotting stores in 4b29e232cae310ac5d117f55b573863f at 1732441257165Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732441257165Flushing stores of testReplayEditsWrittenViaHRegion,,1732441256437.4b29e232cae310ac5d117f55b573863f. at 1732441257165Flushing 4b29e232cae310ac5d117f55b573863f/b: creating writer at 1732441257165Flushing 4b29e232cae310ac5d117f55b573863f/b: appending metadata at 1732441257183 (+18 ms)Flushing 4b29e232cae310ac5d117f55b573863f/b: closing flushed file at 1732441257183Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fdc44d4: reopening flushed file at 1732441257251 (+68 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4b29e232cae310ac5d117f55b573863f in 97ms, sequenceid=32, compaction requested=false; wal=null at 1732441257262 (+11 ms)Cleaning up temporary data from old regions at 1732441257266 (+4 ms)Region opened successfully at 1732441257287 (+21 ms) 2024-11-24T09:40:57,352 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=414 (was 404) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:37776 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:45606 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:46854 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:58704 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:45682 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:42660 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1114 (was 1040) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=590 (was 590), ProcessCount=11 (was 11), AvailableMemoryMB=7727 (was 7749) 2024-11-24T09:40:57,353 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1114 is superior to 1024 2024-11-24T09:40:57,395 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1114, MaxFileDescriptor=1048576, SystemLoadAverage=590, ProcessCount=11, AvailableMemoryMB=7722 2024-11-24T09:40:57,395 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1114 is superior to 1024 2024-11-24T09:40:57,435 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:57,440 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:40:57,441 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:40:57,447 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41969942, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-41969942, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:57,478 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41969942/hregion-41969942.1732441257447, exclude list is [], retry=0 2024-11-24T09:40:57,482 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:57,485 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:57,492 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:57,600 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41969942/hregion-41969942.1732441257447 2024-11-24T09:40:57,600 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:40:57,600 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 56ec4f99547e2ed414b4f9527c112139, NAME => 'testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:40:57,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741907_1085 (size=68) 2024-11-24T09:40:57,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741907_1085 (size=68) 2024-11-24T09:40:57,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741907_1085 (size=68) 2024-11-24T09:40:57,666 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:57,669 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,673 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName a 2024-11-24T09:40:57,673 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,676 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,676 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,680 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName b 2024-11-24T09:40:57,680 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,681 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,682 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,684 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName c 2024-11-24T09:40:57,684 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,685 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,685 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,686 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,687 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,689 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,689 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,690 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:57,692 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,715 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:40:57,715 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 56ec4f99547e2ed414b4f9527c112139; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73916289, jitterRate=0.10143853724002838}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:57,717 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 56ec4f99547e2ed414b4f9527c112139: Writing region info on filesystem at 1732441257666Initializing all the Stores at 1732441257668 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257668Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257668Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257668Cleaning up temporary data from old regions at 1732441257689 (+21 ms)Region opened successfully at 1732441257716 (+27 ms) 2024-11-24T09:40:57,717 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 56ec4f99547e2ed414b4f9527c112139, disabling compactions & flushes 2024-11-24T09:40:57,717 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:40:57,717 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:40:57,717 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. after waiting 0 ms 2024-11-24T09:40:57,717 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:40:57,720 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:40:57,720 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 56ec4f99547e2ed414b4f9527c112139: Waiting for close lock at 1732441257717Disabling compacts and flushes for region at 1732441257717Disabling writes for close at 1732441257717Writing region close event to WAL at 1732441257720 (+3 ms)Closed at 1732441257720 2024-11-24T09:40:57,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741906_1084 (size=93) 2024-11-24T09:40:57,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741906_1084 (size=93) 2024-11-24T09:40:57,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741906_1084 (size=93) 2024-11-24T09:40:57,749 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:40:57,749 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41969942:(num 1732441257447) 2024-11-24T09:40:57,750 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:40:57,753 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:40:57,777 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754, exclude list is [], retry=0 2024-11-24T09:40:57,787 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:40:57,788 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:40:57,788 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:40:57,823 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754 2024-11-24T09:40:57,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:40:57,958 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 56ec4f99547e2ed414b4f9527c112139, NAME => 'testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:40:57,967 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,967 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:40:57,967 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,967 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,970 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,977 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName a 2024-11-24T09:40:57,977 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,978 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,979 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,980 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName b 2024-11-24T09:40:57,980 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,981 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,981 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,982 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName c 2024-11-24T09:40:57,982 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:40:57,984 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:40:57,985 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,986 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,989 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,991 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,991 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:57,991 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:40:57,998 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:58,001 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 56ec4f99547e2ed414b4f9527c112139; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63488410, jitterRate=-0.05394896864891052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:40:58,003 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:40:58,004 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 56ec4f99547e2ed414b4f9527c112139: Running coprocessor pre-open hook at 1732441257967Writing region info on filesystem at 1732441257967Initializing all the Stores at 1732441257969 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257969Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257969Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441257969Cleaning up temporary data from old regions at 1732441257991 (+22 ms)Running coprocessor post-open hooks at 1732441258003 (+12 ms)Region opened successfully at 1732441258004 (+1 ms) 2024-11-24T09:40:58,029 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56ec4f99547e2ed414b4f9527c112139 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T09:40:58,031 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:59,031 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:40:59,548 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T09:41:00,032 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:00,937 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T09:41:01,033 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:02,034 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:02,426 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T09:41:02,427 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T09:41:02,429 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T09:41:02,429 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T09:41:02,431 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T09:41:02,431 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T09:41:03,035 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:03,471 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T09:41:04,036 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:05,037 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:06,038 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:07,039 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56ec4f99547e2ed414b4f9527c112139/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:07,042 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 56ec4f99547e2ed414b4f9527c112139: 2024-11-24T09:41:07,042 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:41:07,058 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 56ec4f99547e2ed414b4f9527c112139: 2024-11-24T09:41:07,059 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T09:41:07,059 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 56ec4f99547e2ed414b4f9527c112139, disabling compactions & flushes 2024-11-24T09:41:07,059 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:41:07,059 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:41:07,059 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. after waiting 0 ms 2024-11-24T09:41:07,059 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:41:07,059 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:41:07,059 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. 2024-11-24T09:41:07,059 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 56ec4f99547e2ed414b4f9527c112139: Waiting for close lock at 1732441267059Running coprocessor pre-close hooks at 1732441267059Disabling compacts and flushes for region at 1732441267059Disabling writes for close at 1732441267059Writing region close event to WAL at 1732441267059Running coprocessor post-close hooks at 1732441267059Closed at 1732441267059 2024-11-24T09:41:07,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741908_1086 (size=2398) 2024-11-24T09:41:07,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741908_1086 (size=2398) 2024-11-24T09:41:07,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741908_1086 (size=2398) 2024-11-24T09:41:07,078 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754, size=2.3 K (2398bytes) 2024-11-24T09:41:07,078 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754 2024-11-24T09:41:07,078 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754 after 0ms 2024-11-24T09:41:07,080 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:07,080 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754 took 3ms 2024-11-24T09:41:07,082 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754 so closing down 2024-11-24T09:41:07,082 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:41:07,083 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732441257754.temp 2024-11-24T09:41:07,084 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000004-wal.1732441257754.temp 2024-11-24T09:41:07,085 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:41:07,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741909_1087 (size=1672) 2024-11-24T09:41:07,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741909_1087 (size=1672) 2024-11-24T09:41:07,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741909_1087 (size=1672) 2024-11-24T09:41:07,091 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000004-wal.1732441257754.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T09:41:07,093 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000004-wal.1732441257754.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000026 2024-11-24T09:41:07,093 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-11-24T09:41:07,093 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754, journal: Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754, size=2.3 K (2398bytes) at 1732441267078Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754 so closing down at 1732441267082 (+4 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000004-wal.1732441257754.temp at 1732441267084 (+2 ms)3 split writer threads finished at 1732441267085 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000004-wal.1732441257754.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732441267091 (+6 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000004-wal.1732441257754.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000026 at 1732441267093 (+2 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1732441267093 2024-11-24T09:41:07,095 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441257754 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441257754 2024-11-24T09:41:07,095 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000026 2024-11-24T09:41:07,096 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:41:07,097 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:07,113 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441267098, exclude list is [], retry=0 2024-11-24T09:41:07,115 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:07,116 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:07,116 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:07,117 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441267098 2024-11-24T09:41:07,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:41:07,118 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 56ec4f99547e2ed414b4f9527c112139, NAME => 'testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:41:07,119 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,119 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:07,119 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,119 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,121 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,121 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName a 2024-11-24T09:41:07,122 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:07,122 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:07,122 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,123 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName b 2024-11-24T09:41:07,123 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:07,124 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:07,124 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,124 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ec4f99547e2ed414b4f9527c112139 columnFamilyName c 2024-11-24T09:41:07,125 DEBUG [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:07,125 INFO [StoreOpener-56ec4f99547e2ed414b4f9527c112139-1 {}] regionserver.HStore(327): Store=56ec4f99547e2ed414b4f9527c112139/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:07,125 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,126 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,127 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,128 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000026 2024-11-24T09:41:07,130 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:07,131 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000026 2024-11-24T09:41:07,131 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56ec4f99547e2ed414b4f9527c112139 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T09:41:07,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/a/cb54113eeb944d1fbe01b6f1d6f06a5e is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732441267051/Put/seqid=0 2024-11-24T09:41:07,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741911_1089 (size=5523) 2024-11-24T09:41:07,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741911_1089 (size=5523) 2024-11-24T09:41:07,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741911_1089 (size=5523) 2024-11-24T09:41:07,153 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/a/cb54113eeb944d1fbe01b6f1d6f06a5e 2024-11-24T09:41:07,172 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/b/ef93a0c2749f4f678b9fc30f00a37852 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732441267043/Put/seqid=0 2024-11-24T09:41:07,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741912_1090 (size=5524) 2024-11-24T09:41:07,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741912_1090 (size=5524) 2024-11-24T09:41:07,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741912_1090 (size=5524) 2024-11-24T09:41:07,179 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/b/ef93a0c2749f4f678b9fc30f00a37852 2024-11-24T09:41:07,198 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/c/ee2159174d604fa8b9f0d9bcd27b64db is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732441267047/Put/seqid=0 2024-11-24T09:41:07,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741913_1091 (size=5457) 2024-11-24T09:41:07,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741913_1091 (size=5457) 2024-11-24T09:41:07,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741913_1091 (size=5457) 2024-11-24T09:41:07,206 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/c/ee2159174d604fa8b9f0d9bcd27b64db 2024-11-24T09:41:07,211 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/a/cb54113eeb944d1fbe01b6f1d6f06a5e as hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/a/cb54113eeb944d1fbe01b6f1d6f06a5e 2024-11-24T09:41:07,217 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/a/cb54113eeb944d1fbe01b6f1d6f06a5e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T09:41:07,218 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/b/ef93a0c2749f4f678b9fc30f00a37852 as hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/b/ef93a0c2749f4f678b9fc30f00a37852 2024-11-24T09:41:07,224 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/b/ef93a0c2749f4f678b9fc30f00a37852, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T09:41:07,225 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/.tmp/c/ee2159174d604fa8b9f0d9bcd27b64db as hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/c/ee2159174d604fa8b9f0d9bcd27b64db 2024-11-24T09:41:07,231 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/c/ee2159174d604fa8b9f0d9bcd27b64db, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T09:41:07,231 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 56ec4f99547e2ed414b4f9527c112139 in 100ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T09:41:07,232 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/0000000000000000026 2024-11-24T09:41:07,233 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,233 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,234 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:41:07,235 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,237 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsAfterAbortingFlush/56ec4f99547e2ed414b4f9527c112139/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T09:41:07,238 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 56ec4f99547e2ed414b4f9527c112139; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70760830, jitterRate=0.05441853404045105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:41:07,238 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56ec4f99547e2ed414b4f9527c112139 2024-11-24T09:41:07,239 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 56ec4f99547e2ed414b4f9527c112139: Running coprocessor pre-open hook at 1732441267119Writing region info on filesystem at 1732441267119Initializing all the Stores at 1732441267120 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441267120Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441267120Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441267120Obtaining lock to block concurrent updates at 1732441267132 (+12 ms)Preparing flush snapshotting stores in 56ec4f99547e2ed414b4f9527c112139 at 1732441267132Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732441267132Flushing stores of testReplayEditsAfterAbortingFlush,,1732441257436.56ec4f99547e2ed414b4f9527c112139. at 1732441267132Flushing 56ec4f99547e2ed414b4f9527c112139/a: creating writer at 1732441267132Flushing 56ec4f99547e2ed414b4f9527c112139/a: appending metadata at 1732441267145 (+13 ms)Flushing 56ec4f99547e2ed414b4f9527c112139/a: closing flushed file at 1732441267145Flushing 56ec4f99547e2ed414b4f9527c112139/b: creating writer at 1732441267158 (+13 ms)Flushing 56ec4f99547e2ed414b4f9527c112139/b: appending metadata at 1732441267171 (+13 ms)Flushing 56ec4f99547e2ed414b4f9527c112139/b: closing flushed file at 1732441267172 (+1 ms)Flushing 56ec4f99547e2ed414b4f9527c112139/c: creating writer at 1732441267183 (+11 ms)Flushing 56ec4f99547e2ed414b4f9527c112139/c: appending metadata at 1732441267198 (+15 ms)Flushing 56ec4f99547e2ed414b4f9527c112139/c: closing flushed file at 1732441267198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16e726e4: reopening flushed file at 1732441267210 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7978e222: reopening flushed file at 1732441267217 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ac74257: reopening flushed file at 1732441267224 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 56ec4f99547e2ed414b4f9527c112139 in 100ms, sequenceid=26, compaction requested=false; wal=null at 1732441267231 (+7 ms)Cleaning up temporary data from old regions at 1732441267233 (+2 ms)Running coprocessor post-open hooks at 1732441267238 (+5 ms)Region opened successfully at 1732441267239 (+1 ms) 2024-11-24T09:41:07,261 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=417 (was 414) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:39538 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:60982 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:59990 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:60988 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:60006 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:39514 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1184 (was 1114) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=589 (was 590), ProcessCount=11 (was 11), AvailableMemoryMB=7601 (was 7722) 2024-11-24T09:41:07,261 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-24T09:41:07,272 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1184, MaxFileDescriptor=1048576, SystemLoadAverage=589, ProcessCount=11, AvailableMemoryMB=7599 2024-11-24T09:41:07,272 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-24T09:41:07,286 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:41:07,288 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:41:07,288 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:41:07,291 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-88550079, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-88550079, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:07,311 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-88550079/hregion-88550079.1732441267292, exclude list is [], retry=0 2024-11-24T09:41:07,314 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:07,314 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:07,314 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:07,316 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-88550079/hregion-88550079.1732441267292 2024-11-24T09:41:07,318 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:41:07,318 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f741f6607f297a81ae4f989777bd44f3, NAME => 'testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:41:07,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741915_1093 (size=61) 2024-11-24T09:41:07,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741915_1093 (size=61) 2024-11-24T09:41:07,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741915_1093 (size=61) 2024-11-24T09:41:07,329 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:07,331 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,332 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f741f6607f297a81ae4f989777bd44f3 columnFamilyName a 2024-11-24T09:41:07,332 DEBUG [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:07,333 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.HStore(327): Store=f741f6607f297a81ae4f989777bd44f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:07,333 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,334 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,334 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,334 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,334 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,336 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,339 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:41:07,339 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f741f6607f297a81ae4f989777bd44f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75284673, jitterRate=0.12182904779911041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:41:07,340 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f741f6607f297a81ae4f989777bd44f3: Writing region info on filesystem at 1732441267329Initializing all the Stores at 1732441267330 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441267330Cleaning up temporary data from old regions at 1732441267334 (+4 ms)Region opened successfully at 1732441267340 (+6 ms) 2024-11-24T09:41:07,340 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f741f6607f297a81ae4f989777bd44f3, disabling compactions & flushes 2024-11-24T09:41:07,340 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,340 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,340 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. after waiting 0 ms 2024-11-24T09:41:07,340 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,340 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,341 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f741f6607f297a81ae4f989777bd44f3: Waiting for close lock at 1732441267340Disabling compacts and flushes for region at 1732441267340Disabling writes for close at 1732441267340Writing region close event to WAL at 1732441267340Closed at 1732441267340 2024-11-24T09:41:07,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741914_1092 (size=93) 2024-11-24T09:41:07,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741914_1092 (size=93) 2024-11-24T09:41:07,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741914_1092 (size=93) 2024-11-24T09:41:07,345 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:41:07,345 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-88550079:(num 1732441267292) 2024-11-24T09:41:07,345 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:41:07,348 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:07,361 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348, exclude list is [], retry=0 2024-11-24T09:41:07,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:07,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:07,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:07,366 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348 2024-11-24T09:41:07,366 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:41:07,366 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f741f6607f297a81ae4f989777bd44f3, NAME => 'testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:41:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,371 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,372 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f741f6607f297a81ae4f989777bd44f3 columnFamilyName a 2024-11-24T09:41:07,372 DEBUG [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:07,372 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.HStore(327): Store=f741f6607f297a81ae4f989777bd44f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:07,373 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,373 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,375 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,375 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,377 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,377 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f741f6607f297a81ae4f989777bd44f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71598414, jitterRate=0.06689950823783875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:41:07,378 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f741f6607f297a81ae4f989777bd44f3: Writing region info on filesystem at 1732441267367Initializing all the Stores at 1732441267368 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441267368Cleaning up temporary data from old regions at 1732441267375 (+7 ms)Region opened successfully at 1732441267378 (+3 ms) 2024-11-24T09:41:07,386 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f741f6607f297a81ae4f989777bd44f3, disabling compactions & flushes 2024-11-24T09:41:07,386 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,386 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,386 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. after waiting 0 ms 2024-11-24T09:41:07,386 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,387 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,387 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. 2024-11-24T09:41:07,387 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f741f6607f297a81ae4f989777bd44f3: Waiting for close lock at 1732441267386Disabling compacts and flushes for region at 1732441267386Disabling writes for close at 1732441267386Writing region close event to WAL at 1732441267387 (+1 ms)Closed at 1732441267387 2024-11-24T09:41:07,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741916_1094 (size=838) 2024-11-24T09:41:07,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741916_1094 (size=838) 2024-11-24T09:41:07,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741916_1094 (size=838) 2024-11-24T09:41:07,405 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348, size=838 (838bytes) 2024-11-24T09:41:07,405 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348 2024-11-24T09:41:07,406 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348 after 1ms 2024-11-24T09:41:07,408 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:07,408 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348 took 3ms 2024-11-24T09:41:07,410 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348 so closing down 2024-11-24T09:41:07,410 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:41:07,411 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441267348.temp 2024-11-24T09:41:07,412 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000003-wal.1732441267348.temp 2024-11-24T09:41:07,413 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:41:07,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741917_1095 (size=838) 2024-11-24T09:41:07,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741917_1095 (size=838) 2024-11-24T09:41:07,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741917_1095 (size=838) 2024-11-24T09:41:07,419 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000003-wal.1732441267348.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T09:41:07,420 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000003-wal.1732441267348.temp to hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000012 2024-11-24T09:41:07,420 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348, size=838, length=838, corrupted=false, cancelled=false 2024-11-24T09:41:07,420 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348, journal: Splitting hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348, size=838 (838bytes) at 1732441267405Finishing writing output for hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348 so closing down at 1732441267410 (+5 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000003-wal.1732441267348.temp at 1732441267412 (+2 ms)3 split writer threads finished at 1732441267413 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000003-wal.1732441267348.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732441267419 (+6 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000003-wal.1732441267348.temp to hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000012 at 1732441267420 (+1 ms)Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348, size=838, length=838, corrupted=false, cancelled=false at 1732441267420 2024-11-24T09:41:07,421 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267348 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441267348 2024-11-24T09:41:07,422 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000012 2024-11-24T09:41:07,425 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:07,808 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:41:07,810 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:07,823 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267811, exclude list is [], retry=0 2024-11-24T09:41:07,825 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:07,825 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:07,826 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:07,828 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267811 2024-11-24T09:41:07,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:41:07,828 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f741f6607f297a81ae4f989777bd44f3, NAME => 'testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:41:07,829 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:07,829 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,829 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,832 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,834 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f741f6607f297a81ae4f989777bd44f3 columnFamilyName a 2024-11-24T09:41:07,834 DEBUG [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:07,835 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.HStore(327): Store=f741f6607f297a81ae4f989777bd44f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:07,835 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,836 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,838 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000012 2024-11-24T09:41:07,841 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:07,842 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000012 2024-11-24T09:41:07,842 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f741f6607f297a81ae4f989777bd44f3 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T09:41:07,866 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/.tmp/a/bcf63f33b2714efbbd43ec6f14176a5e is 79, key is testDatalossWhenInputError/a:x0/1732441267378/Put/seqid=0 2024-11-24T09:41:07,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741919_1097 (size=5808) 2024-11-24T09:41:07,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741919_1097 (size=5808) 2024-11-24T09:41:07,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741919_1097 (size=5808) 2024-11-24T09:41:07,878 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/.tmp/a/bcf63f33b2714efbbd43ec6f14176a5e 2024-11-24T09:41:07,888 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/.tmp/a/bcf63f33b2714efbbd43ec6f14176a5e as hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/a/bcf63f33b2714efbbd43ec6f14176a5e 2024-11-24T09:41:07,897 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/a/bcf63f33b2714efbbd43ec6f14176a5e, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T09:41:07,897 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f741f6607f297a81ae4f989777bd44f3 in 55ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T09:41:07,898 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/0000000000000000012 2024-11-24T09:41:07,899 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,899 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,902 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,904 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T09:41:07,904 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f741f6607f297a81ae4f989777bd44f3; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69753024, jitterRate=0.03940105438232422}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:41:07,905 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f741f6607f297a81ae4f989777bd44f3: Writing region info on filesystem at 1732441267829Initializing all the Stores at 1732441267831 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441267832 (+1 ms)Obtaining lock to block concurrent updates at 1732441267842 (+10 ms)Preparing flush snapshotting stores in f741f6607f297a81ae4f989777bd44f3 at 1732441267842Finished memstore snapshotting testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732441267842Flushing stores of testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3. at 1732441267842Flushing f741f6607f297a81ae4f989777bd44f3/a: creating writer at 1732441267842Flushing f741f6607f297a81ae4f989777bd44f3/a: appending metadata at 1732441267866 (+24 ms)Flushing f741f6607f297a81ae4f989777bd44f3/a: closing flushed file at 1732441267866Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@461942e5: reopening flushed file at 1732441267887 (+21 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f741f6607f297a81ae4f989777bd44f3 in 55ms, sequenceid=12, compaction requested=false; wal=null at 1732441267897 (+10 ms)Cleaning up temporary data from old regions at 1732441267899 (+2 ms)Region opened successfully at 1732441267905 (+6 ms) 2024-11-24T09:41:07,908 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f741f6607f297a81ae4f989777bd44f3, NAME => 'testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:41:07,908 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732441267286.f741f6607f297a81ae4f989777bd44f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:07,908 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,908 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,909 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,910 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f741f6607f297a81ae4f989777bd44f3 columnFamilyName a 2024-11-24T09:41:07,910 DEBUG [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:07,914 DEBUG [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/a/bcf63f33b2714efbbd43ec6f14176a5e 2024-11-24T09:41:07,914 INFO [StoreOpener-f741f6607f297a81ae4f989777bd44f3-1 {}] regionserver.HStore(327): Store=f741f6607f297a81ae4f989777bd44f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:07,915 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,915 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,916 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,916 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,917 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,918 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f741f6607f297a81ae4f989777bd44f3 2024-11-24T09:41:07,920 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testDatalossWhenInputError/f741f6607f297a81ae4f989777bd44f3/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T09:41:07,921 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f741f6607f297a81ae4f989777bd44f3; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69094627, jitterRate=0.029590174555778503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:41:07,921 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f741f6607f297a81ae4f989777bd44f3: Writing region info on filesystem at 1732441267908Initializing all the Stores at 1732441267909 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441267909Cleaning up temporary data from old regions at 1732441267917 (+8 ms)Region opened successfully at 1732441267921 (+4 ms) 2024-11-24T09:41:07,940 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=427 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:39608 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:60076 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:32814 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:60988 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-321562900-172.17.0.2-1732441227025:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:60006 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:39514 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1266 (was 1184) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=566 (was 589), ProcessCount=11 (was 11), AvailableMemoryMB=7513 (was 7599) 2024-11-24T09:41:07,940 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1266 is superior to 1024 2024-11-24T09:41:07,954 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=427, OpenFileDescriptor=1266, MaxFileDescriptor=1048576, SystemLoadAverage=566, ProcessCount=11, AvailableMemoryMB=7512 2024-11-24T09:41:07,955 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1266 is superior to 1024 2024-11-24T09:41:07,971 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:41:07,972 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:41:07,973 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:41:07,975 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-94510294, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-94510294, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:07,988 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-94510294/hregion-94510294.1732441267975, exclude list is [], retry=0 2024-11-24T09:41:07,991 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:07,991 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:07,991 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:07,993 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-94510294/hregion-94510294.1732441267975 2024-11-24T09:41:07,993 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173)] 2024-11-24T09:41:07,993 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5cd73e08edc1b4b620623d2609513481, NAME => 'testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:41:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741921_1099 (size=63) 2024-11-24T09:41:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741921_1099 (size=63) 2024-11-24T09:41:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741921_1099 (size=63) 2024-11-24T09:41:08,003 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:08,004 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,005 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName a 2024-11-24T09:41:08,005 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:08,005 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:08,006 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,006 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName b 2024-11-24T09:41:08,007 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:08,007 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:08,007 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,008 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName c 2024-11-24T09:41:08,008 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:08,009 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:08,009 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,009 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,010 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,010 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,010 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,011 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:41:08,012 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,014 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:41:08,014 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5cd73e08edc1b4b620623d2609513481; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75380114, jitterRate=0.12325122952461243}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:41:08,015 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5cd73e08edc1b4b620623d2609513481: Writing region info on filesystem at 1732441268003Initializing all the Stores at 1732441268003Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441268003Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441268004 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441268004Cleaning up temporary data from old regions at 1732441268010 (+6 ms)Region opened successfully at 1732441268014 (+4 ms) 2024-11-24T09:41:08,015 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5cd73e08edc1b4b620623d2609513481, disabling compactions & flushes 2024-11-24T09:41:08,015 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:08,015 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:08,015 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. after waiting 0 ms 2024-11-24T09:41:08,015 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:08,015 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:08,015 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5cd73e08edc1b4b620623d2609513481: Waiting for close lock at 1732441268015Disabling compacts and flushes for region at 1732441268015Disabling writes for close at 1732441268015Writing region close event to WAL at 1732441268015Closed at 1732441268015 2024-11-24T09:41:08,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741920_1098 (size=93) 2024-11-24T09:41:08,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741920_1098 (size=93) 2024-11-24T09:41:08,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741920_1098 (size=93) 2024-11-24T09:41:08,022 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:41:08,022 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-94510294:(num 1732441267975) 2024-11-24T09:41:08,022 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:41:08,023 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:08,035 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024, exclude list is [], retry=0 2024-11-24T09:41:08,038 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:08,038 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:08,038 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:08,040 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 2024-11-24T09:41:08,040 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:41:08,040 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cd73e08edc1b4b620623d2609513481, NAME => 'testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:41:08,040 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:08,040 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,040 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,041 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,042 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName a 2024-11-24T09:41:08,042 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:08,043 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:08,043 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,044 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName b 2024-11-24T09:41:08,044 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:08,044 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:08,044 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,045 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName c 2024-11-24T09:41:08,045 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:08,046 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:08,046 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,047 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,048 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,049 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,049 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,049 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:41:08,050 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:08,051 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5cd73e08edc1b4b620623d2609513481; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74682290, jitterRate=0.11285284161567688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:41:08,052 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5cd73e08edc1b4b620623d2609513481: Writing region info on filesystem at 1732441268040Initializing all the Stores at 1732441268041 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441268041Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441268041Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441268041Cleaning up temporary data from old regions at 1732441268049 (+8 ms)Region opened successfully at 1732441268052 (+3 ms) 2024-11-24T09:41:08,056 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732441268055/Put/seqid=0 2024-11-24T09:41:08,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741923_1101 (size=4875) 2024-11-24T09:41:08,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741923_1101 (size=4875) 2024-11-24T09:41:08,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741923_1101 (size=4875) 2024-11-24T09:41:08,064 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732441268064/Put/seqid=0 2024-11-24T09:41:08,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741924_1102 (size=4875) 2024-11-24T09:41:08,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741924_1102 (size=4875) 2024-11-24T09:41:08,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741924_1102 (size=4875) 2024-11-24T09:41:08,076 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732441268076/Put/seqid=0 2024-11-24T09:41:08,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741925_1103 (size=4875) 2024-11-24T09:41:08,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741925_1103 (size=4875) 2024-11-24T09:41:08,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741925_1103 (size=4875) 2024-11-24T09:41:08,083 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 5cd73e08edc1b4b620623d2609513481/a 2024-11-24T09:41:08,086 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T09:41:08,086 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T09:41:08,086 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 5cd73e08edc1b4b620623d2609513481/a 2024-11-24T09:41:08,091 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T09:41:08,091 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T09:41:08,091 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 5cd73e08edc1b4b620623d2609513481/a 2024-11-24T09:41:08,095 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T09:41:08,095 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T09:41:08,095 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5cd73e08edc1b4b620623d2609513481 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T09:41:08,109 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/.tmp/a/8ec19c6b62b24f96980cf8b33b520d69 is 55, key is testCompactedBulkLoadedFiles/a:a/1732441268052/Put/seqid=0 2024-11-24T09:41:08,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741926_1104 (size=5107) 2024-11-24T09:41:08,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741926_1104 (size=5107) 2024-11-24T09:41:08,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741926_1104 (size=5107) 2024-11-24T09:41:08,116 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/.tmp/a/8ec19c6b62b24f96980cf8b33b520d69 2024-11-24T09:41:08,121 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/.tmp/a/8ec19c6b62b24f96980cf8b33b520d69 as hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/8ec19c6b62b24f96980cf8b33b520d69 2024-11-24T09:41:08,126 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/8ec19c6b62b24f96980cf8b33b520d69, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T09:41:08,127 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 5cd73e08edc1b4b620623d2609513481 in 32ms, sequenceid=4, compaction requested=false 2024-11-24T09:41:08,127 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5cd73e08edc1b4b620623d2609513481: 2024-11-24T09:41:08,128 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_ 2024-11-24T09:41:08,129 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_ 2024-11-24T09:41:08,130 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_ 2024-11-24T09:41:08,130 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile0 into 5cd73e08edc1b4b620623d2609513481/a as hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_ - updating store file list. 2024-11-24T09:41:08,135 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ad6421323ff141febaa26de82d4ad3f5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:41:08,135 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_ into 5cd73e08edc1b4b620623d2609513481/a 2024-11-24T09:41:08,135 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile0 into 5cd73e08edc1b4b620623d2609513481/a (new location: hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_) 2024-11-24T09:41:08,136 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile1 into 5cd73e08edc1b4b620623d2609513481/a as hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_ - updating store file list. 2024-11-24T09:41:08,140 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:41:08,140 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_ into 5cd73e08edc1b4b620623d2609513481/a 2024-11-24T09:41:08,140 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile1 into 5cd73e08edc1b4b620623d2609513481/a (new location: hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_) 2024-11-24T09:41:08,142 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile2 into 5cd73e08edc1b4b620623d2609513481/a as hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_ - updating store file list. 2024-11-24T09:41:08,146 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 34823243455b4a77b4896c6d5ba1ed54_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:41:08,146 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_ into 5cd73e08edc1b4b620623d2609513481/a 2024-11-24T09:41:08,146 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43013/hbase/testCompactedBulkLoadedFiles/hfile2 into 5cd73e08edc1b4b620623d2609513481/a (new location: hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_) 2024-11-24T09:41:08,153 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T09:41:08,153 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5cd73e08edc1b4b620623d2609513481/a is initiating major compaction (all files) 2024-11-24T09:41:08,153 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5cd73e08edc1b4b620623d2609513481/a in testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:08,153 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/8ec19c6b62b24f96980cf8b33b520d69, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_] into tmpdir=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/.tmp, totalSize=19.3 K 2024-11-24T09:41:08,154 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8ec19c6b62b24f96980cf8b33b520d69, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732441268052 2024-11-24T09:41:08,154 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ad6421323ff141febaa26de82d4ad3f5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T09:41:08,155 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T09:41:08,155 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 34823243455b4a77b4896c6d5ba1ed54_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T09:41:08,167 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/.tmp/a/12425cb7a2db4f71a4f585f647c5f23c is 55, key is testCompactedBulkLoadedFiles/a:a/1732441268052/Put/seqid=0 2024-11-24T09:41:08,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741927_1105 (size=6154) 2024-11-24T09:41:08,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741927_1105 (size=6154) 2024-11-24T09:41:08,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741927_1105 (size=6154) 2024-11-24T09:41:08,179 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/.tmp/a/12425cb7a2db4f71a4f585f647c5f23c as hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/12425cb7a2db4f71a4f585f647c5f23c 2024-11-24T09:41:08,185 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 5cd73e08edc1b4b620623d2609513481/a of 5cd73e08edc1b4b620623d2609513481 into 12425cb7a2db4f71a4f585f647c5f23c(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T09:41:08,185 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5cd73e08edc1b4b620623d2609513481: 2024-11-24T09:41:08,185 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T09:41:08,185 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T09:41:08,216 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024, size=0 (0bytes) 2024-11-24T09:41:08,216 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 might be still open, length is 0 2024-11-24T09:41:08,216 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 2024-11-24T09:41:08,217 WARN [IPC Server handler 2 on default port 43013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-24T09:41:08,217 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 after 1ms 2024-11-24T09:41:09,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:39632 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:45595:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39632 dst: /127.0.0.1:45595 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45595 remote=/127.0.0.1:39632]. Total timeout mills is 60000, 58260 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:09,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:60104 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43599:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60104 dst: /127.0.0.1:43599 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:09,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:32854 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:45495:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32854 dst: /127.0.0.1:45495 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:09,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741922_1106 (size=1368) 2024-11-24T09:41:09,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741922_1106 (size=1368) 2024-11-24T09:41:09,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741922_1106 (size=1368) 2024-11-24T09:41:12,219 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 after 4003ms 2024-11-24T09:41:12,226 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:12,226 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 took 4010ms 2024-11-24T09:41:12,229 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024; continuing. 2024-11-24T09:41:12,229 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 so closing down 2024-11-24T09:41:12,229 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:41:12,231 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441268024.temp 2024-11-24T09:41:12,233 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000003-wal.1732441268024.temp 2024-11-24T09:41:12,233 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:41:12,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741928_1107 (size=635) 2024-11-24T09:41:12,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741928_1107 (size=635) 2024-11-24T09:41:12,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741928_1107 (size=635) 2024-11-24T09:41:12,241 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000003-wal.1732441268024.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T09:41:12,243 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000003-wal.1732441268024.temp to hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000008 2024-11-24T09:41:12,243 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T09:41:12,243 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024, journal: Splitting hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024, size=0 (0bytes) at 1732441268216Finishing writing output for hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 so closing down at 1732441272229 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000003-wal.1732441268024.temp at 1732441272233 (+4 ms)3 split writer threads finished at 1732441272233Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000003-wal.1732441268024.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732441272241 (+8 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000003-wal.1732441268024.temp to hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000008 at 1732441272243 (+2 ms)Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024, size=0, length=0, corrupted=false, cancelled=false at 1732441272243 2024-11-24T09:41:12,245 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441268024 2024-11-24T09:41:12,246 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000008 2024-11-24T09:41:12,246 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:41:12,248 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:12,260 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441272249, exclude list is [], retry=0 2024-11-24T09:41:12,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:12,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:12,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:12,264 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441272249 2024-11-24T09:41:12,264 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:41:12,264 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cd73e08edc1b4b620623d2609513481, NAME => 'testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:41:12,265 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:12,265 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,265 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,266 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,267 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName a 2024-11-24T09:41:12,267 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,274 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/12425cb7a2db4f71a4f585f647c5f23c 2024-11-24T09:41:12,277 DEBUG [StoreFileOpener-5cd73e08edc1b4b620623d2609513481-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:41:12,277 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_ 2024-11-24T09:41:12,280 DEBUG [StoreFileOpener-5cd73e08edc1b4b620623d2609513481-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 34823243455b4a77b4896c6d5ba1ed54_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:41:12,280 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_ 2024-11-24T09:41:12,285 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/8ec19c6b62b24f96980cf8b33b520d69 2024-11-24T09:41:12,288 DEBUG [StoreFileOpener-5cd73e08edc1b4b620623d2609513481-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ad6421323ff141febaa26de82d4ad3f5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T09:41:12,288 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_ 2024-11-24T09:41:12,288 WARN [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@714e7cc0 2024-11-24T09:41:12,289 WARN [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@714e7cc0 2024-11-24T09:41:12,289 WARN [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/8ec19c6b62b24f96980cf8b33b520d69 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@714e7cc0 2024-11-24T09:41:12,289 WARN [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@714e7cc0 2024-11-24T09:41:12,289 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/8ec19c6b62b24f96980cf8b33b520d69, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_] to archive 2024-11-24T09:41:12,290 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T09:41:12,292 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_ to hdfs://localhost:43013/hbase/archive/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_ 2024-11-24T09:41:12,293 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_ to hdfs://localhost:43013/hbase/archive/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/34823243455b4a77b4896c6d5ba1ed54_SeqId_4_ 2024-11-24T09:41:12,294 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/8ec19c6b62b24f96980cf8b33b520d69 to hdfs://localhost:43013/hbase/archive/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/8ec19c6b62b24f96980cf8b33b520d69 2024-11-24T09:41:12,296 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_ to hdfs://localhost:43013/hbase/archive/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/a/ad6421323ff141febaa26de82d4ad3f5_SeqId_4_ 2024-11-24T09:41:12,296 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,296 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,297 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName b 2024-11-24T09:41:12,297 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,297 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,297 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,298 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cd73e08edc1b4b620623d2609513481 columnFamilyName c 2024-11-24T09:41:12,298 DEBUG [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,298 INFO [StoreOpener-5cd73e08edc1b4b620623d2609513481-1 {}] regionserver.HStore(327): Store=5cd73e08edc1b4b620623d2609513481/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,298 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,299 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,300 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,301 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000008 2024-11-24T09:41:12,303 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:12,305 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 5cd73e08edc1b4b620623d2609513481 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "5cd73e08edc1b4b620623d2609513481" family_name: "a" compaction_input: "8ec19c6b62b24f96980cf8b33b520d69" compaction_input: "ad6421323ff141febaa26de82d4ad3f5_SeqId_4_" compaction_input: "1fed860332ff444aa831a2bbe5cb0b1b_SeqId_4_" compaction_input: "34823243455b4a77b4896c6d5ba1ed54_SeqId_4_" compaction_output: "12425cb7a2db4f71a4f585f647c5f23c" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T09:41:12,305 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T09:41:12,305 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000008 2024-11-24T09:41:12,306 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/0000000000000000008 2024-11-24T09:41:12,307 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,307 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,307 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:41:12,309 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5cd73e08edc1b4b620623d2609513481 2024-11-24T09:41:12,311 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testCompactedBulkLoadedFiles/5cd73e08edc1b4b620623d2609513481/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T09:41:12,312 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5cd73e08edc1b4b620623d2609513481; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66324848, jitterRate=-0.011682748794555664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:41:12,312 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5cd73e08edc1b4b620623d2609513481: Writing region info on filesystem at 1732441272265Initializing all the Stores at 1732441272266 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272266Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272266Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272266Cleaning up temporary data from old regions at 1732441272307 (+41 ms)Region opened successfully at 1732441272312 (+5 ms) 2024-11-24T09:41:12,314 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5cd73e08edc1b4b620623d2609513481, disabling compactions & flushes 2024-11-24T09:41:12,315 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:12,315 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:12,315 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. after waiting 0 ms 2024-11-24T09:41:12,315 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:12,316 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732441267971.5cd73e08edc1b4b620623d2609513481. 2024-11-24T09:41:12,316 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5cd73e08edc1b4b620623d2609513481: Waiting for close lock at 1732441272314Disabling compacts and flushes for region at 1732441272314Disabling writes for close at 1732441272315 (+1 ms)Writing region close event to WAL at 1732441272315Closed at 1732441272315 2024-11-24T09:41:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741929_1108 (size=93) 2024-11-24T09:41:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741929_1108 (size=93) 2024-11-24T09:41:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741929_1108 (size=93) 2024-11-24T09:41:12,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:41:12,325 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441272249) 2024-11-24T09:41:12,338 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=435 (was 427) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1926524479_22 at /127.0.0.1:55130 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (373550922) connection to localhost/127.0.0.1:43013 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:43013 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1926524479_22 at /127.0.0.1:34880 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1345 (was 1266) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=566 (was 566), ProcessCount=11 (was 11), AvailableMemoryMB=7506 (was 7512) 2024-11-24T09:41:12,338 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1345 is superior to 1024 2024-11-24T09:41:12,350 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=435, OpenFileDescriptor=1345, MaxFileDescriptor=1048576, SystemLoadAverage=566, ProcessCount=11, AvailableMemoryMB=7506 2024-11-24T09:41:12,350 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1345 is superior to 1024 2024-11-24T09:41:12,363 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:41:12,364 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:41:12,365 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:41:12,367 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-76677509, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/hregion-76677509, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:12,379 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-76677509/hregion-76677509.1732441272368, exclude list is [], retry=0 2024-11-24T09:41:12,382 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:12,382 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:12,382 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:12,384 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-76677509/hregion-76677509.1732441272368 2024-11-24T09:41:12,384 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043),(127.0.0.1/127.0.0.1:34113:34113)] 2024-11-24T09:41:12,384 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 769226cc02dc156b2475fbf20417dc82, NAME => 'testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43013/hbase 2024-11-24T09:41:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741931_1110 (size=67) 2024-11-24T09:41:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741931_1110 (size=67) 2024-11-24T09:41:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741931_1110 (size=67) 2024-11-24T09:41:12,399 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:12,401 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,402 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName a 2024-11-24T09:41:12,402 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,403 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,403 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,404 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName b 2024-11-24T09:41:12,404 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,405 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,405 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,407 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName c 2024-11-24T09:41:12,407 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,407 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,407 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,409 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,410 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,410 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:41:12,412 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,414 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:41:12,415 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 769226cc02dc156b2475fbf20417dc82; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74484769, jitterRate=0.1099095493555069}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:41:12,415 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 769226cc02dc156b2475fbf20417dc82: Writing region info on filesystem at 1732441272399Initializing all the Stores at 1732441272400 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272400Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272400Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272400Cleaning up temporary data from old regions at 1732441272410 (+10 ms)Region opened successfully at 1732441272415 (+5 ms) 2024-11-24T09:41:12,415 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 769226cc02dc156b2475fbf20417dc82, disabling compactions & flushes 2024-11-24T09:41:12,415 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,415 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,415 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. after waiting 0 ms 2024-11-24T09:41:12,415 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,416 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,416 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 769226cc02dc156b2475fbf20417dc82: Waiting for close lock at 1732441272415Disabling compacts and flushes for region at 1732441272415Disabling writes for close at 1732441272415Writing region close event to WAL at 1732441272416 (+1 ms)Closed at 1732441272416 2024-11-24T09:41:12,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741930_1109 (size=93) 2024-11-24T09:41:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741930_1109 (size=93) 2024-11-24T09:41:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741930_1109 (size=93) 2024-11-24T09:41:12,422 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:41:12,422 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-76677509:(num 1732441272368) 2024-11-24T09:41:12,422 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:41:12,424 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:12,437 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424, exclude list is [], retry=0 2024-11-24T09:41:12,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:12,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:12,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:12,441 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424 2024-11-24T09:41:12,442 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:41:12,442 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 769226cc02dc156b2475fbf20417dc82, NAME => 'testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:41:12,442 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:12,442 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,442 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,443 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,444 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName a 2024-11-24T09:41:12,444 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,445 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,445 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,445 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName b 2024-11-24T09:41:12,446 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,446 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,446 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,447 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName c 2024-11-24T09:41:12,447 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,447 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,447 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,448 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,449 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,450 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,450 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,450 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:41:12,452 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,453 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 769226cc02dc156b2475fbf20417dc82; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73155102, jitterRate=0.09009596705436707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:41:12,453 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 769226cc02dc156b2475fbf20417dc82: Writing region info on filesystem at 1732441272442Initializing all the Stores at 1732441272443 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272443Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272443Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272443Cleaning up temporary data from old regions at 1732441272450 (+7 ms)Region opened successfully at 1732441272453 (+3 ms) 2024-11-24T09:41:12,461 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 769226cc02dc156b2475fbf20417dc82 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T09:41:12,481 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/a/fc1bdd5345f04c718bbfa18512a6d6cb is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732441272453/Put/seqid=0 2024-11-24T09:41:12,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741933_1112 (size=5958) 2024-11-24T09:41:12,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741933_1112 (size=5958) 2024-11-24T09:41:12,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741933_1112 (size=5958) 2024-11-24T09:41:12,488 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/a/fc1bdd5345f04c718bbfa18512a6d6cb 2024-11-24T09:41:12,493 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/a/fc1bdd5345f04c718bbfa18512a6d6cb as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/a/fc1bdd5345f04c718bbfa18512a6d6cb 2024-11-24T09:41:12,499 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/a/fc1bdd5345f04c718bbfa18512a6d6cb, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T09:41:12,500 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 769226cc02dc156b2475fbf20417dc82 in 39ms, sequenceid=13, compaction requested=false 2024-11-24T09:41:12,500 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 769226cc02dc156b2475fbf20417dc82: 2024-11-24T09:41:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 769226cc02dc156b2475fbf20417dc82, disabling compactions & flushes 2024-11-24T09:41:12,515 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. after waiting 0 ms 2024-11-24T09:41:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,516 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,516 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:12,516 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 769226cc02dc156b2475fbf20417dc82: Waiting for close lock at 1732441272515Disabling compacts and flushes for region at 1732441272515Disabling writes for close at 1732441272515Writing region close event to WAL at 1732441272516 (+1 ms)Closed at 1732441272516 2024-11-24T09:41:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741932_1111 (size=2805) 2024-11-24T09:41:12,519 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424 not finished, retry = 0 2024-11-24T09:41:12,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741932_1111 (size=2805) 2024-11-24T09:41:12,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741932_1111 (size=2805) 2024-11-24T09:41:12,564 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T09:41:12,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T09:41:12,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T09:41:12,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T09:41:12,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T09:41:12,637 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424, size=2.7 K (2805bytes) 2024-11-24T09:41:12,637 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424 2024-11-24T09:41:12,637 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424 after 0ms 2024-11-24T09:41:12,639 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:12,640 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424 took 3ms 2024-11-24T09:41:12,642 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424 so closing down 2024-11-24T09:41:12,642 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:41:12,643 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732441272424.temp 2024-11-24T09:41:12,644 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000003-wal.1732441272424.temp 2024-11-24T09:41:12,644 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:41:12,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741934_1113 (size=2312) 2024-11-24T09:41:12,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741934_1113 (size=2312) 2024-11-24T09:41:12,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741934_1113 (size=2312) 2024-11-24T09:41:12,653 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000003-wal.1732441272424.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T09:41:12,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000003-wal.1732441272424.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000035 2024-11-24T09:41:12,655 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-11-24T09:41:12,655 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424, journal: Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424, size=2.7 K (2805bytes) at 1732441272637Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424 so closing down at 1732441272642 (+5 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000003-wal.1732441272424.temp at 1732441272644 (+2 ms)3 split writer threads finished at 1732441272644Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000003-wal.1732441272424.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732441272653 (+9 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000003-wal.1732441272424.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000035 at 1732441272655 (+2 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1732441272655 2024-11-24T09:41:12,657 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272424 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441272424 2024-11-24T09:41:12,658 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000035 2024-11-24T09:41:12,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:41:12,659 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:12,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660, exclude list is [], retry=0 2024-11-24T09:41:12,673 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:12,674 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:12,674 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:12,675 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 2024-11-24T09:41:12,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:41:12,676 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 769226cc02dc156b2475fbf20417dc82, NAME => 'testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:41:12,676 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:12,676 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,676 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,677 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,678 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName a 2024-11-24T09:41:12,678 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,683 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/a/fc1bdd5345f04c718bbfa18512a6d6cb 2024-11-24T09:41:12,683 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,683 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,684 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName b 2024-11-24T09:41:12,684 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,685 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,685 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,685 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName c 2024-11-24T09:41:12,685 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:12,686 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:12,686 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,686 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,688 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,688 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000035 2024-11-24T09:41:12,690 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:12,692 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000035 2024-11-24T09:41:12,692 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 769226cc02dc156b2475fbf20417dc82 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T09:41:12,706 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/b/9157beb7ae3541fe9374de6833389795 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732441272500/Put/seqid=0 2024-11-24T09:41:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741936_1115 (size=5958) 2024-11-24T09:41:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741936_1115 (size=5958) 2024-11-24T09:41:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741936_1115 (size=5958) 2024-11-24T09:41:12,713 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/b/9157beb7ae3541fe9374de6833389795 2024-11-24T09:41:12,732 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/c/0facd1f628c94c5ca1d84ea11326ab66 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732441272506/Put/seqid=0 2024-11-24T09:41:12,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741937_1116 (size=5958) 2024-11-24T09:41:12,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741937_1116 (size=5958) 2024-11-24T09:41:12,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741937_1116 (size=5958) 2024-11-24T09:41:12,738 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/c/0facd1f628c94c5ca1d84ea11326ab66 2024-11-24T09:41:12,743 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/b/9157beb7ae3541fe9374de6833389795 as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/b/9157beb7ae3541fe9374de6833389795 2024-11-24T09:41:12,747 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/b/9157beb7ae3541fe9374de6833389795, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T09:41:12,749 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/c/0facd1f628c94c5ca1d84ea11326ab66 as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/c/0facd1f628c94c5ca1d84ea11326ab66 2024-11-24T09:41:12,753 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/c/0facd1f628c94c5ca1d84ea11326ab66, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T09:41:12,753 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 769226cc02dc156b2475fbf20417dc82 in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T09:41:12,754 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000035 2024-11-24T09:41:12,754 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,754 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,755 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:41:12,756 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:12,758 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T09:41:12,759 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 769226cc02dc156b2475fbf20417dc82; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68593764, jitterRate=0.022126734256744385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:41:12,759 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 769226cc02dc156b2475fbf20417dc82: Writing region info on filesystem at 1732441272676Initializing all the Stores at 1732441272677 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272677Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272677Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441272677Obtaining lock to block concurrent updates at 1732441272692 (+15 ms)Preparing flush snapshotting stores in 769226cc02dc156b2475fbf20417dc82 at 1732441272692Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732441272692Flushing stores of testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. at 1732441272692Flushing 769226cc02dc156b2475fbf20417dc82/b: creating writer at 1732441272692Flushing 769226cc02dc156b2475fbf20417dc82/b: appending metadata at 1732441272706 (+14 ms)Flushing 769226cc02dc156b2475fbf20417dc82/b: closing flushed file at 1732441272706Flushing 769226cc02dc156b2475fbf20417dc82/c: creating writer at 1732441272717 (+11 ms)Flushing 769226cc02dc156b2475fbf20417dc82/c: appending metadata at 1732441272731 (+14 ms)Flushing 769226cc02dc156b2475fbf20417dc82/c: closing flushed file at 1732441272731Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29c57237: reopening flushed file at 1732441272742 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@246377ae: reopening flushed file at 1732441272748 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 769226cc02dc156b2475fbf20417dc82 in 61ms, sequenceid=35, compaction requested=false; wal=null at 1732441272753 (+5 ms)Cleaning up temporary data from old regions at 1732441272754 (+1 ms)Region opened successfully at 1732441272759 (+5 ms) 2024-11-24T09:41:12,818 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660, size=0 (0bytes) 2024-11-24T09:41:12,818 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 might be still open, length is 0 2024-11-24T09:41:12,818 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 2024-11-24T09:41:12,819 WARN [IPC Server handler 2 on default port 43013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-24T09:41:12,819 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 after 1ms 2024-11-24T09:41:15,800 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:55210 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43599:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55210 dst: /127.0.0.1:43599 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43599 remote=/127.0.0.1:55210]. Total timeout mills is 60000, 56984 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:15,801 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:34938 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:45595:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34938 dst: /127.0.0.1:45595 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:15,802 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_54813548_22 at /127.0.0.1:36804 [Receiving block BP-321562900-172.17.0.2-1732441227025:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:45495:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36804 dst: /127.0.0.1:45495 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:15,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741935_1117 (size=2304) 2024-11-24T09:41:15,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741935_1117 (size=2304) 2024-11-24T09:41:15,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741935_1117 (size=2304) 2024-11-24T09:41:16,749 INFO [master/ba35ada4783f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T09:41:16,749 INFO [master/ba35ada4783f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T09:41:16,821 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 after 4002ms 2024-11-24T09:41:16,828 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:16,828 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 took 4010ms 2024-11-24T09:41:16,831 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660; continuing. 2024-11-24T09:41:16,831 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 so closing down 2024-11-24T09:41:16,831 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T09:41:16,832 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732441272660.temp 2024-11-24T09:41:16,834 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000037-wal.1732441272660.temp 2024-11-24T09:41:16,834 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T09:41:16,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741938_1118 (size=2312) 2024-11-24T09:41:16,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741938_1118 (size=2312) 2024-11-24T09:41:16,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741938_1118 (size=2312) 2024-11-24T09:41:16,844 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000037-wal.1732441272660.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T09:41:16,845 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000037-wal.1732441272660.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000066 2024-11-24T09:41:16,845 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T09:41:16,845 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660, journal: Splitting hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660, size=0 (0bytes) at 1732441272818Finishing writing output for hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 so closing down at 1732441276831 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000037-wal.1732441272660.temp at 1732441276834 (+3 ms)3 split writer threads finished at 1732441276834Closed recovered edits writer path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000037-wal.1732441272660.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732441276844 (+10 ms)Rename recovered edits hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000037-wal.1732441272660.temp to hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000066 at 1732441276845 (+1 ms)Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660, size=0, length=0, corrupted=false, cancelled=false at 1732441276845 2024-11-24T09:41:16,846 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 to hdfs://localhost:43013/hbase/oldWALs/wal.1732441272660 2024-11-24T09:41:16,847 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000066 2024-11-24T09:41:16,847 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T09:41:16,849 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43013/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362, archiveDir=hdfs://localhost:43013/hbase/oldWALs, maxLogs=32 2024-11-24T09:41:16,861 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441276849, exclude list is [], retry=0 2024-11-24T09:41:16,863 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43599,DS-6e28db07-2977-48d6-98a2-40409d0bf5d3,DISK] 2024-11-24T09:41:16,863 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45595,DS-8e140724-d6a2-44a3-8d6b-394c73ae2a29,DISK] 2024-11-24T09:41:16,863 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45495,DS-c38566f6-ad2a-4a7a-83c5-fff68116226e,DISK] 2024-11-24T09:41:16,865 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441276849 2024-11-24T09:41:16,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34113:34113),(127.0.0.1/127.0.0.1:33173:33173),(127.0.0.1/127.0.0.1:39043:39043)] 2024-11-24T09:41:16,865 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:41:16,867 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,867 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName a 2024-11-24T09:41:16,868 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:16,872 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/a/fc1bdd5345f04c718bbfa18512a6d6cb 2024-11-24T09:41:16,872 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:16,872 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,873 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName b 2024-11-24T09:41:16,873 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:16,878 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/b/9157beb7ae3541fe9374de6833389795 2024-11-24T09:41:16,878 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:16,878 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,879 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 769226cc02dc156b2475fbf20417dc82 columnFamilyName c 2024-11-24T09:41:16,879 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:41:16,883 DEBUG [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/c/0facd1f628c94c5ca1d84ea11326ab66 2024-11-24T09:41:16,883 INFO [StoreOpener-769226cc02dc156b2475fbf20417dc82-1 {}] regionserver.HStore(327): Store=769226cc02dc156b2475fbf20417dc82/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:41:16,884 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,884 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,886 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,886 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000066 2024-11-24T09:41:16,888 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T09:41:16,892 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000066 2024-11-24T09:41:16,892 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 769226cc02dc156b2475fbf20417dc82 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T09:41:16,906 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/a/6452a3d5f50648459c47aa277a435ce3 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732441272764/Put/seqid=0 2024-11-24T09:41:16,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741940_1120 (size=5958) 2024-11-24T09:41:16,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741940_1120 (size=5958) 2024-11-24T09:41:16,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741940_1120 (size=5958) 2024-11-24T09:41:16,913 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/a/6452a3d5f50648459c47aa277a435ce3 2024-11-24T09:41:16,929 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/b/660b42aee1814ca7aef9c9cfbd234c7f is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732441272771/Put/seqid=0 2024-11-24T09:41:16,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741941_1121 (size=5958) 2024-11-24T09:41:16,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741941_1121 (size=5958) 2024-11-24T09:41:16,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741941_1121 (size=5958) 2024-11-24T09:41:16,935 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/b/660b42aee1814ca7aef9c9cfbd234c7f 2024-11-24T09:41:16,954 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/c/c2391dcb377c4070bcba99a8bf7a5e0a is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732441272778/Put/seqid=0 2024-11-24T09:41:16,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741942_1122 (size=5958) 2024-11-24T09:41:16,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741942_1122 (size=5958) 2024-11-24T09:41:16,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741942_1122 (size=5958) 2024-11-24T09:41:16,961 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/c/c2391dcb377c4070bcba99a8bf7a5e0a 2024-11-24T09:41:16,965 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/a/6452a3d5f50648459c47aa277a435ce3 as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/a/6452a3d5f50648459c47aa277a435ce3 2024-11-24T09:41:16,970 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/a/6452a3d5f50648459c47aa277a435ce3, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T09:41:16,970 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/b/660b42aee1814ca7aef9c9cfbd234c7f as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/b/660b42aee1814ca7aef9c9cfbd234c7f 2024-11-24T09:41:16,974 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/b/660b42aee1814ca7aef9c9cfbd234c7f, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T09:41:16,975 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/.tmp/c/c2391dcb377c4070bcba99a8bf7a5e0a as hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/c/c2391dcb377c4070bcba99a8bf7a5e0a 2024-11-24T09:41:16,979 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/c/c2391dcb377c4070bcba99a8bf7a5e0a, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T09:41:16,979 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 769226cc02dc156b2475fbf20417dc82 in 87ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T09:41:16,980 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/0000000000000000066 2024-11-24T09:41:16,981 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,981 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,981 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T09:41:16,982 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 769226cc02dc156b2475fbf20417dc82 2024-11-24T09:41:16,984 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/hbase/data/default/testReplayEditsWrittenViaHRegion/769226cc02dc156b2475fbf20417dc82/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T09:41:16,985 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 769226cc02dc156b2475fbf20417dc82; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63268308, jitterRate=-0.05722874402999878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T09:41:16,985 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 769226cc02dc156b2475fbf20417dc82: Writing region info on filesystem at 1732441276865Initializing all the Stores at 1732441276866 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441276866Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441276866Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732441276866Obtaining lock to block concurrent updates at 1732441276892 (+26 ms)Preparing flush snapshotting stores in 769226cc02dc156b2475fbf20417dc82 at 1732441276892Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732441276892Flushing stores of testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. at 1732441276893 (+1 ms)Flushing 769226cc02dc156b2475fbf20417dc82/a: creating writer at 1732441276893Flushing 769226cc02dc156b2475fbf20417dc82/a: appending metadata at 1732441276906 (+13 ms)Flushing 769226cc02dc156b2475fbf20417dc82/a: closing flushed file at 1732441276906Flushing 769226cc02dc156b2475fbf20417dc82/b: creating writer at 1732441276916 (+10 ms)Flushing 769226cc02dc156b2475fbf20417dc82/b: appending metadata at 1732441276929 (+13 ms)Flushing 769226cc02dc156b2475fbf20417dc82/b: closing flushed file at 1732441276929Flushing 769226cc02dc156b2475fbf20417dc82/c: creating writer at 1732441276939 (+10 ms)Flushing 769226cc02dc156b2475fbf20417dc82/c: appending metadata at 1732441276954 (+15 ms)Flushing 769226cc02dc156b2475fbf20417dc82/c: closing flushed file at 1732441276954Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13ceae79: reopening flushed file at 1732441276964 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47473f99: reopening flushed file at 1732441276970 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45ef6b6f: reopening flushed file at 1732441276975 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 769226cc02dc156b2475fbf20417dc82 in 87ms, sequenceid=66, compaction requested=false; wal=null at 1732441276979 (+4 ms)Cleaning up temporary data from old regions at 1732441276981 (+2 ms)Region opened successfully at 1732441276985 (+4 ms) 2024-11-24T09:41:16,995 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 769226cc02dc156b2475fbf20417dc82, disabling compactions & flushes 2024-11-24T09:41:16,995 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:16,995 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:16,996 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. after waiting 0 ms 2024-11-24T09:41:16,996 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:16,997 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732441272363.769226cc02dc156b2475fbf20417dc82. 2024-11-24T09:41:16,997 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 769226cc02dc156b2475fbf20417dc82: Waiting for close lock at 1732441276995Disabling compacts and flushes for region at 1732441276995Disabling writes for close at 1732441276996 (+1 ms)Writing region close event to WAL at 1732441276997 (+1 ms)Closed at 1732441276997 2024-11-24T09:41:16,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741939_1119 (size=93) 2024-11-24T09:41:17,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741939_1119 (size=93) 2024-11-24T09:41:17,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741939_1119 (size=93) 2024-11-24T09:41:17,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T09:41:17,001 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732441276849) 2024-11-24T09:41:17,016 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=442 (was 435) Potentially hanging thread: IPC Client (373550922) connection to localhost/127.0.0.1:43013 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1023720833_22 at /127.0.0.1:36834 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:43013 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39539 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (373550922) connection to localhost/127.0.0.1:39539 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1023720833_22 at /127.0.0.1:55234 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1023720833_22 at /127.0.0.1:35000 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1345) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 566), ProcessCount=11 (was 11), AvailableMemoryMB=7494 (was 7506) 2024-11-24T09:41:17,016 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-24T09:41:17,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T09:41:17,016 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T09:41:17,016 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:41:17,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:41:17,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:41:17,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T09:41:17,017 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T09:41:17,017 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=633453724, stopped=false 2024-11-24T09:41:17,017 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba35ada4783f,43385,1732441231965 2024-11-24T09:41:17,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T09:41:17,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T09:41:17,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T09:41:17,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:41:17,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:41:17,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:41:17,044 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T09:41:17,045 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:41:17,045 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:41:17,045 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:41:17,045 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T09:41:17,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:41:17,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:41:17,046 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba35ada4783f,36811,1732441233150' ***** 2024-11-24T09:41:17,046 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T09:41:17,046 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba35ada4783f,35917,1732441233385' ***** 2024-11-24T09:41:17,046 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T09:41:17,047 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T09:41:17,047 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T09:41:17,047 INFO [RS:2;ba35ada4783f:35917 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T09:41:17,047 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T09:41:17,047 INFO [RS:2;ba35ada4783f:35917 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(3091): Received CLOSE for a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:41:17,047 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(959): stopping server ba35ada4783f,35917,1732441233385 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(959): stopping server ba35ada4783f,36811,1732441233150 2024-11-24T09:41:17,047 INFO [RS:2;ba35ada4783f:35917 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba35ada4783f:36811. 2024-11-24T09:41:17,047 INFO [RS:2;ba35ada4783f:35917 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ba35ada4783f:35917. 2024-11-24T09:41:17,047 DEBUG [RS:0;ba35ada4783f:36811 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:41:17,047 DEBUG [RS:2;ba35ada4783f:35917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:41:17,047 DEBUG [RS:0;ba35ada4783f:36811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:41:17,047 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a47f41b92e82354ab3fd0381bbcca2bb, disabling compactions & flushes 2024-11-24T09:41:17,047 DEBUG [RS:2;ba35ada4783f:35917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:41:17,047 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:41:17,047 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(976): stopping server ba35ada4783f,35917,1732441233385; all regions closed. 2024-11-24T09:41:17,047 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T09:41:17,047 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T09:41:17,047 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. after waiting 0 ms 2024-11-24T09:41:17,048 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T09:41:17,048 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:41:17,048 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T09:41:17,049 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T09:41:17,049 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, a47f41b92e82354ab3fd0381bbcca2bb=testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb.} 2024-11-24T09:41:17,049 DEBUG [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a47f41b92e82354ab3fd0381bbcca2bb 2024-11-24T09:41:17,049 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T09:41:17,049 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T09:41:17,049 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T09:41:17,049 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T09:41:17,049 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T09:41:17,049 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-24T09:41:17,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741833_1009 (size=93) 2024-11-24T09:41:17,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741833_1009 (size=93) 2024-11-24T09:41:17,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741833_1009 (size=93) 2024-11-24T09:41:17,056 DEBUG [RS:2;ba35ada4783f:35917 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs 2024-11-24T09:41:17,056 INFO [RS:2;ba35ada4783f:35917 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba35ada4783f%2C35917%2C1732441233385:(num 1732441235464) 2024-11-24T09:41:17,056 DEBUG [RS:2;ba35ada4783f:35917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:41:17,056 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:41:17,056 INFO [RS:2;ba35ada4783f:35917 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:41:17,057 INFO [RS:2;ba35ada4783f:35917 {}] hbase.ChoreService(370): Chore service for: regionserver/ba35ada4783f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T09:41:17,057 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T09:41:17,057 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T09:41:17,057 INFO [regionserver/ba35ada4783f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:41:17,057 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T09:41:17,057 INFO [RS:2;ba35ada4783f:35917 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:41:17,057 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/default/testReplayEditsAfterRegionMovedWithMultiCF/a47f41b92e82354ab3fd0381bbcca2bb/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T09:41:17,057 INFO [RS:2;ba35ada4783f:35917 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35917 2024-11-24T09:41:17,058 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/info/49fc3771e15446c39778c0fea6e4d443 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb./info:regioninfo/1732441255811/Put/seqid=0 2024-11-24T09:41:17,058 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:41:17,058 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a47f41b92e82354ab3fd0381bbcca2bb: Waiting for close lock at 1732441277047Running coprocessor pre-close hooks at 1732441277047Disabling compacts and flushes for region at 1732441277047Disabling writes for close at 1732441277048 (+1 ms)Writing region close event to WAL at 1732441277049 (+1 ms)Running coprocessor post-close hooks at 1732441277058 (+9 ms)Closed at 1732441277058 2024-11-24T09:41:17,058 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732441251253.a47f41b92e82354ab3fd0381bbcca2bb. 2024-11-24T09:41:17,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741943_1123 (size=6778) 2024-11-24T09:41:17,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741943_1123 (size=6778) 2024-11-24T09:41:17,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741943_1123 (size=6778) 2024-11-24T09:41:17,064 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/info/49fc3771e15446c39778c0fea6e4d443 2024-11-24T09:41:17,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba35ada4783f,35917,1732441233385 2024-11-24T09:41:17,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:41:17,065 INFO [RS:2;ba35ada4783f:35917 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:41:17,069 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/.tmp/info/49fc3771e15446c39778c0fea6e4d443 as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/info/49fc3771e15446c39778c0fea6e4d443 2024-11-24T09:41:17,074 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba35ada4783f,35917,1732441233385] 2024-11-24T09:41:17,074 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/info/49fc3771e15446c39778c0fea6e4d443, entries=8, sequenceid=23, filesize=6.6 K 2024-11-24T09:41:17,075 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=23, compaction requested=false 2024-11-24T09:41:17,080 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-24T09:41:17,080 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T09:41:17,080 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T09:41:17,080 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732441277049Running coprocessor pre-close hooks at 1732441277049Disabling compacts and flushes for region at 1732441277049Disabling writes for close at 1732441277049Obtaining lock to block concurrent updates at 1732441277049Preparing flush snapshotting stores in 1588230740 at 1732441277049Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732441277050 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732441277053 (+3 ms)Flushing 1588230740/info: creating writer at 1732441277053Flushing 1588230740/info: appending metadata at 1732441277057 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732441277057Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@463ce343: reopening flushed file at 1732441277068 (+11 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=23, compaction requested=false at 1732441277075 (+7 ms)Writing region close event to WAL at 1732441277076 (+1 ms)Running coprocessor post-close hooks at 1732441277080 (+4 ms)Closed at 1732441277080 2024-11-24T09:41:17,081 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T09:41:17,082 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba35ada4783f,35917,1732441233385 already deleted, retry=false 2024-11-24T09:41:17,082 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba35ada4783f,35917,1732441233385 expired; onlineServers=1 2024-11-24T09:41:17,087 INFO [regionserver/ba35ada4783f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:41:17,087 INFO [regionserver/ba35ada4783f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:41:17,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:41:17,174 INFO [RS:2;ba35ada4783f:35917 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:41:17,174 INFO [RS:2;ba35ada4783f:35917 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba35ada4783f,35917,1732441233385; zookeeper connection closed. 2024-11-24T09:41:17,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35917-0x1016d80e1650003, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:41:17,175 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@23f14ae3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@23f14ae3 2024-11-24T09:41:17,249 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(976): stopping server ba35ada4783f,36811,1732441233150; all regions closed. 2024-11-24T09:41:17,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741893_1071 (size=1552) 2024-11-24T09:41:17,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741893_1071 (size=1552) 2024-11-24T09:41:17,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741893_1071 (size=1552) 2024-11-24T09:41:17,260 DEBUG [RS:0;ba35ada4783f:36811 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs 2024-11-24T09:41:17,260 INFO [RS:0;ba35ada4783f:36811 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba35ada4783f%2C36811%2C1732441233150.meta:.meta(num 1732441254833) 2024-11-24T09:41:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741835_1011 (size=2652) 2024-11-24T09:41:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741835_1011 (size=2652) 2024-11-24T09:41:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741835_1011 (size=2652) 2024-11-24T09:41:17,265 DEBUG [RS:0;ba35ada4783f:36811 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/oldWALs 2024-11-24T09:41:17,265 INFO [RS:0;ba35ada4783f:36811 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba35ada4783f%2C36811%2C1732441233150:(num 1732441235525) 2024-11-24T09:41:17,265 DEBUG [RS:0;ba35ada4783f:36811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:41:17,265 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:41:17,265 INFO [RS:0;ba35ada4783f:36811 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:41:17,265 INFO [RS:0;ba35ada4783f:36811 {}] hbase.ChoreService(370): Chore service for: regionserver/ba35ada4783f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T09:41:17,265 INFO [RS:0;ba35ada4783f:36811 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:41:17,265 INFO [regionserver/ba35ada4783f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:41:17,266 INFO [RS:0;ba35ada4783f:36811 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36811 2024-11-24T09:41:17,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba35ada4783f,36811,1732441233150 2024-11-24T09:41:17,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:41:17,290 INFO [RS:0;ba35ada4783f:36811 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:41:17,298 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba35ada4783f,36811,1732441233150] 2024-11-24T09:41:17,307 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba35ada4783f,36811,1732441233150 already deleted, retry=false 2024-11-24T09:41:17,307 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba35ada4783f,36811,1732441233150 expired; onlineServers=0 2024-11-24T09:41:17,307 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba35ada4783f,43385,1732441231965' ***** 2024-11-24T09:41:17,307 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T09:41:17,308 INFO [M:0;ba35ada4783f:43385 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:41:17,308 INFO [M:0;ba35ada4783f:43385 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:41:17,308 DEBUG [M:0;ba35ada4783f:43385 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T09:41:17,308 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T09:41:17,308 DEBUG [M:0;ba35ada4783f:43385 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T09:41:17,308 DEBUG [master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732441234978 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732441234978,5,FailOnTimeoutGroup] 2024-11-24T09:41:17,308 DEBUG [master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732441234984 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732441234984,5,FailOnTimeoutGroup] 2024-11-24T09:41:17,309 INFO [M:0;ba35ada4783f:43385 {}] hbase.ChoreService(370): Chore service for: master/ba35ada4783f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T09:41:17,309 INFO [M:0;ba35ada4783f:43385 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:41:17,309 DEBUG [M:0;ba35ada4783f:43385 {}] master.HMaster(1795): Stopping service threads 2024-11-24T09:41:17,309 INFO [M:0;ba35ada4783f:43385 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T09:41:17,310 INFO [M:0;ba35ada4783f:43385 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T09:41:17,311 INFO [M:0;ba35ada4783f:43385 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T09:41:17,311 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T09:41:17,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T09:41:17,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:41:17,319 DEBUG [M:0;ba35ada4783f:43385 {}] zookeeper.ZKUtil(347): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T09:41:17,319 WARN [M:0;ba35ada4783f:43385 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T09:41:17,321 INFO [M:0;ba35ada4783f:43385 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/.lastflushedseqids 2024-11-24T09:41:17,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741944_1124 (size=119) 2024-11-24T09:41:17,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741944_1124 (size=119) 2024-11-24T09:41:17,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741944_1124 (size=119) 2024-11-24T09:41:17,333 INFO [M:0;ba35ada4783f:43385 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T09:41:17,333 INFO [M:0;ba35ada4783f:43385 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T09:41:17,333 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T09:41:17,333 INFO [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:41:17,333 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:41:17,333 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T09:41:17,333 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:41:17,333 INFO [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-11-24T09:41:17,348 DEBUG [M:0;ba35ada4783f:43385 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/afd4b09d43004e3da966f10c353a04a8 is 82, key is hbase:meta,,1/info:regioninfo/1732441255061/Put/seqid=0 2024-11-24T09:41:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741945_1125 (size=6063) 2024-11-24T09:41:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741945_1125 (size=6063) 2024-11-24T09:41:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741945_1125 (size=6063) 2024-11-24T09:41:17,354 INFO [M:0;ba35ada4783f:43385 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/afd4b09d43004e3da966f10c353a04a8 2024-11-24T09:41:17,373 DEBUG [M:0;ba35ada4783f:43385 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36dcfb725b224a6a893627111b87c787 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732441251729/Put/seqid=0 2024-11-24T09:41:17,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741946_1126 (size=7908) 2024-11-24T09:41:17,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741946_1126 (size=7908) 2024-11-24T09:41:17,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741946_1126 (size=7908) 2024-11-24T09:41:17,380 INFO [M:0;ba35ada4783f:43385 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36dcfb725b224a6a893627111b87c787 2024-11-24T09:41:17,384 INFO [M:0;ba35ada4783f:43385 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 36dcfb725b224a6a893627111b87c787 2024-11-24T09:41:17,398 DEBUG [M:0;ba35ada4783f:43385 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/45e423ecb91f4417a1e07d3f66b9b4d4 is 69, key is ba35ada4783f,35917,1732441233385/rs:state/1732441235030/Put/seqid=0 2024-11-24T09:41:17,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:41:17,399 INFO [RS:0;ba35ada4783f:36811 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:41:17,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36811-0x1016d80e1650001, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:41:17,399 INFO [RS:0;ba35ada4783f:36811 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba35ada4783f,36811,1732441233150; zookeeper connection closed. 2024-11-24T09:41:17,399 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e735863 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e735863 2024-11-24T09:41:17,399 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T09:41:17,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741947_1127 (size=5440) 2024-11-24T09:41:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741947_1127 (size=5440) 2024-11-24T09:41:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741947_1127 (size=5440) 2024-11-24T09:41:17,404 INFO [M:0;ba35ada4783f:43385 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/45e423ecb91f4417a1e07d3f66b9b4d4 2024-11-24T09:41:17,409 INFO [M:0;ba35ada4783f:43385 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45e423ecb91f4417a1e07d3f66b9b4d4 2024-11-24T09:41:17,410 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/afd4b09d43004e3da966f10c353a04a8 as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/afd4b09d43004e3da966f10c353a04a8 2024-11-24T09:41:17,416 INFO [M:0;ba35ada4783f:43385 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/afd4b09d43004e3da966f10c353a04a8, entries=14, sequenceid=207, filesize=5.9 K 2024-11-24T09:41:17,417 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36dcfb725b224a6a893627111b87c787 as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/36dcfb725b224a6a893627111b87c787 2024-11-24T09:41:17,422 INFO [M:0;ba35ada4783f:43385 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 36dcfb725b224a6a893627111b87c787 2024-11-24T09:41:17,422 INFO [M:0;ba35ada4783f:43385 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/36dcfb725b224a6a893627111b87c787, entries=21, sequenceid=207, filesize=7.7 K 2024-11-24T09:41:17,423 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/45e423ecb91f4417a1e07d3f66b9b4d4 as hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/45e423ecb91f4417a1e07d3f66b9b4d4 2024-11-24T09:41:17,427 INFO [M:0;ba35ada4783f:43385 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45e423ecb91f4417a1e07d3f66b9b4d4 2024-11-24T09:41:17,427 INFO [M:0;ba35ada4783f:43385 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43013/user/jenkins/test-data/12ca29e4-041b-96d5-dc7c-6b9f75dfcb63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/45e423ecb91f4417a1e07d3f66b9b4d4, entries=3, sequenceid=207, filesize=5.3 K 2024-11-24T09:41:17,429 INFO [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 95ms, sequenceid=207, compaction requested=false 2024-11-24T09:41:17,430 INFO [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:41:17,430 DEBUG [M:0;ba35ada4783f:43385 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732441277333Disabling compacts and flushes for region at 1732441277333Disabling writes for close at 1732441277333Obtaining lock to block concurrent updates at 1732441277334 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732441277334Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1732441277334Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732441277334Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732441277334Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732441277348 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732441277348Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732441277359 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732441277372 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732441277373 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732441277384 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732441277397 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732441277398 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61ba6511: reopening flushed file at 1732441277409 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd385da: reopening flushed file at 1732441277416 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@232ba3f7: reopening flushed file at 1732441277422 (+6 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 95ms, sequenceid=207, compaction requested=false at 1732441277429 (+7 ms)Writing region close event to WAL at 1732441277430 (+1 ms)Closed at 1732441277430 2024-11-24T09:41:17,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43599 is added to blk_1073741830_1006 (size=86759) 2024-11-24T09:41:17,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741830_1006 (size=86759) 2024-11-24T09:41:17,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45595 is added to blk_1073741830_1006 (size=86759) 2024-11-24T09:41:17,433 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:41:17,433 INFO [M:0;ba35ada4783f:43385 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T09:41:17,433 INFO [M:0;ba35ada4783f:43385 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43385 2024-11-24T09:41:17,433 INFO [M:0;ba35ada4783f:43385 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:41:17,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:41:17,561 INFO [M:0;ba35ada4783f:43385 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:41:17,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43385-0x1016d80e1650000, quorum=127.0.0.1:52387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:41:17,568 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441267098 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441267098 (inode 16688) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732441257434/wal.1732441267098 (inode 16688) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:41:17,571 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441257045 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441257045 (inode 16665) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732441256436/wal.1732441257045 (inode 16665) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:41:17,572 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732441246474/wal.1732441246702 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:17,575 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267811 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267811 (inode 16714) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732441267285/wal.1732441267811 (inode 16714) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:41:17,576 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732441267970/wal.1732441268024 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:17,576 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732441272362/wal.1732441272660 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:17,577 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732441237230/wal.1732441237297 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:41:17,579 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal.1732441237099 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal.1732441237099 (inode 16485) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732441236904/wal.1732441237099 (inode 16485) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:41:17,582 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal.1732441236658 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal.1732441236658 (inode 16462) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732441236438/wal.1732441236658 (inode 16462) Holder DFSClient_NONMAPREDUCE_54813548_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T09:41:17,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@11039a9d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:41:17,660 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7000b5ce{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T09:41:17,660 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T09:41:17,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7701b046{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T09:41:17,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@137d43c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir/,STOPPED} 2024-11-24T09:41:17,665 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T09:41:17,665 WARN [BP-321562900-172.17.0.2-1732441227025 heartbeating to localhost/127.0.0.1:43013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T09:41:17,665 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T09:41:17,665 WARN [BP-321562900-172.17.0.2-1732441227025 heartbeating to localhost/127.0.0.1:43013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-321562900-172.17.0.2-1732441227025 (Datanode Uuid ce27ccbc-32d8-41ca-afbc-75575b160ac7) service to localhost/127.0.0.1:43013 2024-11-24T09:41:17,667 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data5/current/BP-321562900-172.17.0.2-1732441227025 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:41:17,667 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data6/current/BP-321562900-172.17.0.2-1732441227025 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:41:17,668 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T09:41:17,713 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42bdeb7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:41:17,713 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@486e75d4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T09:41:17,713 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T09:41:17,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f53d973{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T09:41:17,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b29fce4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir/,STOPPED} 2024-11-24T09:41:17,716 WARN [BP-321562900-172.17.0.2-1732441227025 heartbeating to localhost/127.0.0.1:43013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T09:41:17,716 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T09:41:17,716 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T09:41:17,716 WARN [BP-321562900-172.17.0.2-1732441227025 heartbeating to localhost/127.0.0.1:43013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-321562900-172.17.0.2-1732441227025 (Datanode Uuid 73904766-b6c2-4e27-9693-9a1e641ebc27) service to localhost/127.0.0.1:43013 2024-11-24T09:41:17,717 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data4/current/BP-321562900-172.17.0.2-1732441227025 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:41:17,717 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data3/current/BP-321562900-172.17.0.2-1732441227025 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:41:17,717 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T09:41:17,723 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cdc8afb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T09:41:17,724 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@796dc018{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T09:41:17,724 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T09:41:17,724 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55114173{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T09:41:17,724 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7bbda1ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir/,STOPPED} 2024-11-24T09:41:17,726 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T09:41:17,726 WARN [BP-321562900-172.17.0.2-1732441227025 heartbeating to localhost/127.0.0.1:43013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T09:41:17,726 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T09:41:17,726 WARN [BP-321562900-172.17.0.2-1732441227025 heartbeating to localhost/127.0.0.1:43013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-321562900-172.17.0.2-1732441227025 (Datanode Uuid 78e58b0a-1a42-4227-bab3-490aa0a195c5) service to localhost/127.0.0.1:43013 2024-11-24T09:41:17,727 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data1/current/BP-321562900-172.17.0.2-1732441227025 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:41:17,727 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/cluster_c0122bae-401b-7e0d-38e7-19ed19090e6d/data/data2/current/BP-321562900-172.17.0.2-1732441227025 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T09:41:17,728 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T09:41:17,737 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T09:41:17,738 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T09:41:17,738 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T09:41:17,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T09:41:17,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23bac8fd-4ed5-6bfc-8871-1a00fbaeb5e4/hadoop.log.dir/,STOPPED} 2024-11-24T09:41:17,762 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T09:41:17,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down